changeset 1241:f1a454e020e5

flow - SequenceConverter - in progress
author Devel 1
date Tue, 30 Jun 2020 16:46:13 +0200
parents f082bcb89613
children 172c6a940a93
files stress-tester/src/main/java/com/passus/st/scanner/SequenceConverter.java stress-tester/src/main/resources/flow_analyzer.py stress-tester/src/test/java/com/passus/st/filter/MatchFilterTest.java
diffstat 3 files changed, 147 insertions(+), 2 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/stress-tester/src/main/java/com/passus/st/scanner/SequenceConverter.java	Tue Jun 30 16:46:13 2020 +0200
@@ -0,0 +1,142 @@
+package com.passus.st.scanner;
+
+import com.google.gson.Gson;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import org.yaml.snakeyaml.DumperOptions;
+import org.yaml.snakeyaml.Yaml;
+
+import static com.passus.commons.collection.FluentBuilder.*;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+
+/**
+ *
+ * @author mikolaj.podbielski
+ */
+public class SequenceConverter {
+
+    enum IdType {
+        IP, Cookie, Header
+    }
+
+    IdType idType = IdType.IP;
+    String idKey;
+    int multiplier = 7;
+    int maxRepeats = 100;
+
+    /*
+    - type: matcher
+      matches:
+          '@req.uri': {$contains: 'test'}
+          '@resp.status.code': 200
+     */
+    static Map<String, Object> prepareMatchFilter(List<String> ignoredUrls) {
+        List<Map<String, Object>> matches = new ArrayList<>(ignoredUrls.size());
+        for (String ignoredUrl : ignoredUrls) {
+            matches.add(smap("@req.uri", ignoredUrl));
+        }
+        return lmap(
+                e("type", "matcher"),
+                //                e("acceptOnMatch", true),
+                e("matches", matches)
+        );
+
+    }
+
+    /*
+  - type: sequence
+    exact: true
+    sequence:
+      - match: {"@req.uri": "/res1", "@resp.getCookie('id')": {$neq: "123"}}
+      - match: {"@req.getHeader('xyz')": "abc"}
+        repeats: 100
+     */
+    Map<String, Object> prepareSequenceFilter(List<String> steps) {
+        List<Map<String, Object>> sequence = new ArrayList<>(steps.size());
+        boolean first = true;
+        for (String step : steps) {
+            Map<String, Object> matcher = lmap(e("@req.uri", step));
+            if (first) {
+                first = false;
+            } else {
+                switch (idType) {
+                    case IP:
+                        matcher.put("@context.session.srcIp", "@_i0.context.session.srcIp");
+                        break;
+                    case Cookie:
+                        matcher.put(String.format("@req.getCookie('%s')", idKey), String.format("@_i0.req.getCookie('%s')", idKey));
+                        break;
+                    case Header:
+                        matcher.put(String.format("@req.getHeader('%s')", idKey), String.format("@_i0.req.getHeader('%s')", idKey));
+                        break;
+                }
+            }
+            sequence.add(lmap(
+                    e("match", matcher),
+                    e("repeats", maxRepeats)
+            ));
+        }
+
+        return lmap(
+                e("type", "sequence"),
+                e("exact", true),
+                e("sequence", sequence),
+                e("action", smap("multiplication", multiplier))
+        );
+    }
+
+    void writeSequenceFilterConf(List<List<String>> sequences, List<String> ignoredUrls, File file) throws IOException {
+        List<Map<String, Object>> filters = new ArrayList<>();
+        if (ignoredUrls != null) {
+            filters.add(prepareMatchFilter(ignoredUrls));
+        }
+        for (List<String> sequence : sequences) {
+            filters.add(prepareSequenceFilter(sequence));
+        }
+        Map<String, Object> cfg = smap("filters", filters);
+
+        DumperOptions opts = new DumperOptions();
+        Yaml yaml = new Yaml();
+
+        try (FileOutputStream fos = new FileOutputStream(file);
+                OutputStreamWriter osw = new OutputStreamWriter(fos, StandardCharsets.UTF_8);) {
+            yaml.dump(cfg, osw);
+        }
+    }
+
+    static List<SeqWithFolds> readFolds(File file, Gson gson) throws IOException {
+        try (FileInputStream fis = new FileInputStream(file);
+                InputStreamReader isr = new InputStreamReader(fis, StandardCharsets.UTF_8)) {
+            SeqWithFolds[] arr = gson.fromJson(isr, SeqWithFolds[].class);
+            return Arrays.asList(arr);
+        }
+    }
+
+    public static final class SeqWithFolds {
+
+        public List<String> sequence;
+        public float prob;
+        public List<List<String>> unfolded;
+    }
+
+    public static void main(String[] args) throws Exception {
+        File input = new File("D:\\hg\\stress-tester\\stress-tester\\src\\main\\resources\\seq_L7f_unfolded.json");
+        List<SeqWithFolds> swf = readFolds(input, new Gson());
+        System.out.println("");
+
+        List<String> ignoredUrls = Arrays.asList("/user/check", "/stacje/podpowiedzi", "/api/mobile");
+        SequenceConverter converter = new SequenceConverter();
+        File output = new File("D:\\hg\\stress-tester\\stress-tester\\x.yaml");
+        converter.writeSequenceFilterConf(swf.get(1).unfolded, ignoredUrls, output);
+    }
+
+}
--- a/stress-tester/src/main/resources/flow_analyzer.py	Tue Jun 30 15:52:22 2020 +0200
+++ b/stress-tester/src/main/resources/flow_analyzer.py	Tue Jun 30 16:46:13 2020 +0200
@@ -10,7 +10,7 @@
 from urllib import parse
 
 
-class flowAnalyzer:
+class FlowAnalyzer:
     def get_files_list(self, pathString="", glob=False, globString="", disallowedSuffixex = [".gz", ".zip"], debug=False):
         """Funkcja, której celem jest przygotowanie listy plików z logami do wczytania
         Może działać na dwa sposoby przez podanie po prostu ścieżki pliku z logami
@@ -695,7 +695,7 @@
         for cleanRule in args.c:
             cleanRules.append(cleanRule)
 
-    analyzer = flowAnalyzer()
+    analyzer = FlowAnalyzer()
     analyzer.get_files_list(pathString=args.input)
     analyzer.get_events_line_csv(dateFormat='%Y-%m-%d_%H_%M_%S_%f') #yyyy-MM-dd_HH_mm_ss_SSS000
     analyzer.prep_data(cleanRules=cleanRules)
--- a/stress-tester/src/test/java/com/passus/st/filter/MatchFilterTest.java	Tue Jun 30 15:52:22 2020 +0200
+++ b/stress-tester/src/test/java/com/passus/st/filter/MatchFilterTest.java	Tue Jun 30 16:46:13 2020 +0200
@@ -15,6 +15,7 @@
 
 import static com.passus.st.utils.Assert.assertNoError;
 import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertFalse;
 import static org.testng.AssertJUnit.assertTrue;
 
 /**
@@ -113,6 +114,7 @@
 
         String filterConfig = "filters:\n"
                 + "    - type: matcher\n"
+                + "      acceptOnMatch: false\n"
                 + "      matches:\n"
                 + "          '@req.uri': {$contains: 'test'}\n"
                 + "          '@resp.status.code': 200\n";
@@ -125,6 +127,7 @@
         assertTrue(filters.get(0) instanceof MatchFilter);
 
         MatchFilter filter = (MatchFilter) filters.get(0);
+        assertFalse(filter.isAcceptOnMatch());
         MessagePredicate predicate = filter.getPredicate();
 
         List<HttpRequest> res = filter(httpRequests, "{'@req.url': {$contains: \"test2\"}}");