Mercurial > stress-tester

changeset 1225:214d097dd288

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
flow - in progress
author Devel 1
date Thu, 25 Jun 2020 14:06:44 +0200
parents cff85e9bf7a8
children 20b9a2c14db3
files stress-tester/src/main/java/com/passus/st/config/CommonNodeDefs.java stress-tester/src/main/java/com/passus/st/scanner/FlowAnalyzerCommand.java stress-tester/src/main/java/com/passus/st/scanner/HttpUrlSequencePayloadAnalyzer.java stress-tester/src/main/resources/flow_analyzer.py
diffstat 4 files changed, 32 insertions(+), 20 deletions(-) [+]
line wrap: on
line diff
--- a/stress-tester/src/main/java/com/passus/st/config/CommonNodeDefs.java	Thu Jun 25 11:50:54 2020 +0200
+++ b/stress-tester/src/main/java/com/passus/st/config/CommonNodeDefs.java	Thu Jun 25 14:06:44 2020 +0200
@@ -1,6 +1,7 @@
 package com.passus.st.config;
 
 import com.passus.config.schema.NodeDefinition;
+import com.passus.config.validation.DoubleValidator;
 import com.passus.config.validation.LongValidator;
 import com.passus.st.filter.MessagePredicateNodeDefinition;
 import com.passus.st.utils.PeriodValueTransformer;
@@ -29,6 +30,8 @@
 
     public static NodeDefinition LONG_GREATER_EQUAL_ZERO_DEF = valueDefLong().addValidator(LongValidator.GREATER_EQUAL_ZERO);
 
+    public static NodeDefinition FLOAT_GREATER_EQUAL_ZERO_DEF = valueDefFloat().addValidator(DoubleValidator.GREATER_EQUAL_ZERO);
+
     public static NodeDefinition MSG_PREDICATE_DEF = new MessagePredicateNodeDefinition();
 
     public static NodeDefinition PERIOD_DEF = valueDef().setTransformer(PeriodValueTransformer.INSTANCE);
--- a/stress-tester/src/main/java/com/passus/st/scanner/FlowAnalyzerCommand.java	Thu Jun 25 11:50:54 2020 +0200
+++ b/stress-tester/src/main/java/com/passus/st/scanner/FlowAnalyzerCommand.java	Thu Jun 25 14:06:44 2020 +0200
@@ -20,6 +20,7 @@
     public static final String DEFAULT_PYTHON_CMD = IS_WINDOWS ? "python" : "python3";
     public static final String DEFAULT_SCRIPT_PATH = "flow_analyzer.py";
     public static final String RESOURCE = "/flow_analyzer.py";
+    public static final float DEFAULT_PMIN = 0.05f;
     public static final int DEFAULT_LMIN = 3;
     public static final int DEFAULT_LMAX = 7;
     public static final boolean DEFAULT_USE_FOLD = false;
@@ -27,23 +28,12 @@
     String pythonCmd = DEFAULT_PYTHON_CMD;
     String scriptPath = DEFAULT_SCRIPT_PATH;
     String dataPath;
+    float pmin = DEFAULT_PMIN;
     int lmin = DEFAULT_LMIN;
     int lmax = DEFAULT_LMAX;
     boolean useFold = DEFAULT_USE_FOLD;
     List<String> cleanRules;
 
-    static void extractEmbeddedScript(File target) throws IOException {
-        target.delete();
-        extractResource(RESOURCE, target);
-    }
-
-    static void extractResource(String resource, File dest) throws IOException {
-        try (InputStream is = FlowAnalyzerCommand.class.getResourceAsStream(resource);) {
-            Path target = Paths.get(dest.toURI());
-            Files.copy(is, target);
-        }
-    }
-
     void run() throws IOException, InterruptedException {
         checkExists(scriptPath, "scriptPath");
         checkExists(dataPath, "dataPath");
@@ -51,7 +41,8 @@
 
         List<String> commandLine = new ArrayList<>();
         commandLine.addAll(Arrays.asList(
-                pythonCmd, scriptPath, dataPath, "-lmin", Integer.toString(lmin), "-lmax", Integer.toString(lmax)
+                pythonCmd, scriptPath, dataPath, "-pmin", Float.toString(pmin),
+                "-lmin", Integer.toString(lmin), "-lmax", Integer.toString(lmax)
         ));
         if (useFold) {
             commandLine.add("--fold");
@@ -94,4 +85,16 @@
         }
     }
 
+    static void extractEmbeddedScript(File target) throws IOException {
+        target.delete();
+        extractResource(RESOURCE, target);
+    }
+
+    static void extractResource(String resource, File dest) throws IOException {
+        try (InputStream is = FlowAnalyzerCommand.class.getResourceAsStream(resource);) {
+            Path target = Paths.get(dest.toURI());
+            Files.copy(is, target);
+        }
+    }
+
 }
--- a/stress-tester/src/main/java/com/passus/st/scanner/HttpUrlSequencePayloadAnalyzer.java	Thu Jun 25 11:50:54 2020 +0200
+++ b/stress-tester/src/main/java/com/passus/st/scanner/HttpUrlSequencePayloadAnalyzer.java	Thu Jun 25 14:06:44 2020 +0200
@@ -38,6 +38,7 @@
 import static com.passus.config.schema.ConfigurationSchemaBuilder.tupleDef;
 import static com.passus.st.Protocols.HTTP;
 import static com.passus.st.config.CommonNodeDefs.BOOLEAN_DEF;
+import static com.passus.st.config.CommonNodeDefs.FLOAT_GREATER_EQUAL_ZERO_DEF;
 import static com.passus.st.config.CommonNodeDefs.INT_GREATER_THAN_ZERO_DEF;
 import static com.passus.st.config.CommonNodeDefs.STRING_DEF;
 import java.util.List;
@@ -58,6 +59,7 @@
 
     private String dataPath;
     private List<String> cleanRules;
+    private float pmin = FlowAnalyzerCommand.DEFAULT_PMIN;
     private int lmin = FlowAnalyzerCommand.DEFAULT_LMIN;
     private int lmax = FlowAnalyzerCommand.DEFAULT_LMAX;
     private boolean useFold;
@@ -73,6 +75,7 @@
         String userIdSource = config.getString("userIdSource", "ip");
         userIdExtractor = resolveUserIdExtractor(userIdSource);
         dataPath = config.getString("dataPath", "sequence_data.csv");
+        pmin = config.getFloat("pmin", FlowAnalyzerCommand.DEFAULT_PMIN);
         lmin = config.getInteger("lmin", FlowAnalyzerCommand.DEFAULT_LMIN);
         lmax = config.getInteger("lmax", FlowAnalyzerCommand.DEFAULT_LMAX);
         useFold = config.getBoolean("useFold", FlowAnalyzerCommand.DEFAULT_USE_FOLD);
@@ -156,6 +159,7 @@
             FlowAnalyzerCommand.extractEmbeddedScript(new File(command.scriptPath));
             command.dataPath = dataPath;
             command.cleanRules = cleanRules;
+            command.pmin = pmin;
             command.lmin = lmin;
             command.lmax = lmax;
             command.useFold = useFold;
@@ -253,6 +257,7 @@
                     tupleDef("dataPath", STRING_DEF).setRequired(false),
                     tupleDef("cleanRules", listDef()).setRequired(false),
                     tupleDef("useFold", BOOLEAN_DEF).setRequired(false),
+                    tupleDef("pmin", FLOAT_GREATER_EQUAL_ZERO_DEF).setRequired(false),
                     tupleDef("lmin", INT_GREATER_THAN_ZERO_DEF).setRequired(false),
                     tupleDef("lmax", INT_GREATER_THAN_ZERO_DEF).setRequired(false)
             );
--- a/stress-tester/src/main/resources/flow_analyzer.py	Thu Jun 25 11:50:54 2020 +0200
+++ b/stress-tester/src/main/resources/flow_analyzer.py	Thu Jun 25 14:06:44 2020 +0200
@@ -460,9 +460,9 @@
             sequences = newSequences
 
         # Normalizujemy i sortujemy
-        self.markov = sorted([(x, y/totalNextSum) for x,y in sequences], reverse=True, key=lambda x:x[1])
-        self.markovMapped = sorted([([self.idsMapRev[int(z)] for z in x], y/totalNextSum) for x,y in sequences], reverse=True, key=lambda x:x[1])
-        self.markov_not_norm = sorted([(x, y) for x,y in sequences], reverse=True, key=lambda x:x[1])
+        self.markov_not_norm = sorted([xy for xy in sequences], reverse=True, key=lambda x: x[1])
+        self.markov = [(x, y / totalNextSum) for x, y in self.markov_not_norm]
+        self.markovMapped = [([self.idsMapRev[int(z)] for z in x], y / totalNextSum) for x, y in self.markov_not_norm]
         return self.markovMapped
 
     def analyze_markov_fold(self, historyLength=1, targetLength=5, shareLimit=0.0001,
@@ -609,10 +609,10 @@
                 sequences = newSequences
 
         # Normalizujemy i sortujemy
-        self.markov = sorted([(x, y / totalNextSum) for x, y in sequences], reverse=True, key=lambda x: x[1])
-        self.markovMapped = sorted([([self.idsMapRev[int(z)] for z in x], y / totalNextSum) for x, y in sequences],
-                                   reverse=True, key=lambda x: x[1])
-        self.markov_not_norm = sorted([(x, y) for x, y in sequences], reverse=True, key=lambda x: x[1])
+        self.markov_not_norm = sorted([xy for xy in sequences], reverse=True, key=lambda x: x[1])
+        self.markov = [(x, y / totalNextSum) for x, y in self.markov_not_norm]
+        self.markovMapped = [([self.idsMapRev[int(z)] for z in x], y / totalNextSum) for x, y in self.markov_not_norm]
+        self.markovLoopsMapped = [([self.idsMapRev[int(x)] for x in xx], [self.idsMapRev[int(y)] for y in yy], z, v) for xx, yy, z, v in self.markovLoops]
         return self.markovMapped
 
 
@@ -622,6 +622,7 @@
     parser = argparse.ArgumentParser('Flow Analyzer')
     parser.add_argument('input')
     parser.add_argument('-c', nargs='+')
+    parser.add_argument('-pmin', type=float, default=0.005)
     parser.add_argument('-lmin', type=int, default=3)
     parser.add_argument('-lmax', type=int, default=7)
     parser.add_argument('--fold', action='store_true')