Mercurial > stress-tester
changeset 1180:33ff26538bfc
Converter improvements in progress
line wrap: on
line diff
--- a/stress-tester/src/main/java/com/passus/st/CliHelper.java Tue Jun 16 16:53:08 2020 +0200 +++ b/stress-tester/src/main/java/com/passus/st/CliHelper.java Wed Jun 17 11:23:46 2020 +0200 @@ -1,22 +1,22 @@ package com.passus.st; import com.passus.commons.Assert; -import com.passus.config.Configuration; -import com.passus.config.ConfigurationContext; -import com.passus.config.YamlConfigurationReader; +import com.passus.commons.plugin.PluginFactory; +import com.passus.config.*; import com.passus.config.validation.Errors; import com.passus.config.validation.ObjectError; import com.passus.filter.ValueExtractor; import com.passus.net.PortRangeSet; import com.passus.net.http.session.HttpSessionAnalyzer; import com.passus.st.client.FlowExecutor; -import com.passus.st.filter.FlowFilter; -import com.passus.st.filter.FlowFiltersConfigurator; import com.passus.st.client.http.filter.HttpVarsFilter; +import com.passus.st.config.JobConfigurator; import com.passus.st.emitter.Emitter; import com.passus.st.emitter.PassThroughSessionMapper; import com.passus.st.emitter.RuleBasedSessionMapper; import com.passus.st.emitter.SessionMapper; +import com.passus.st.filter.FlowFilter; +import com.passus.st.filter.FlowFiltersConfigurator; import com.passus.st.lookup.LookupsDomainConfigurator; import com.passus.st.reader.nc.NcHeader; import com.passus.st.reader.pcap.PcapFileHeader; @@ -28,6 +28,7 @@ import org.apache.commons.io.IOUtils; import java.io.File; +import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; @@ -228,6 +229,36 @@ return Collections.emptyMap(); } + public Configuration readConfiguration(String configFileName) throws IOException, NodeException { + File configFile = new File(configFileName); + if (!configFile.exists()) { + printError("File '" + configFile + "' does not exists."); + } + + return YamlConfigurationReader.readFromFile(configFile); + } + + public JobConfigurator processJobConfiguration(Configuration config, + ConfigurationContext context, + Errors errors, + PluginFactory<DomainConfigurator> pluginFactory) { + JobConfigurator jobConfigurator = new JobConfigurator(); + jobConfigurator.setConfiguratorFactory(pluginFactory); + + jobConfigurator.configure(config, errors, context); + if (errors.hasError()) { + StringBuilder sb = new StringBuilder(); + + errors.getAllErrors().forEach(error -> { + sb.append(error.toString()).append("\n"); + }); + + printError(sb.toString()); + } + + return jobConfigurator; + } + private void handleErrors(Errors errors, File file) { if (errors.getErrorCount() != 0) { printError("Error in file '" + file.getAbsolutePath() + "'.", false);
--- a/stress-tester/src/main/java/com/passus/st/ConverterHttpClient.java Tue Jun 16 16:53:08 2020 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,147 +0,0 @@ -package com.passus.st; - -import com.passus.commons.Assert; -import com.passus.commons.service.Service; -import com.passus.st.client.*; -import com.passus.st.filter.FlowFilter; -import com.passus.st.filter.FlowFilterChain; -import com.passus.st.client.http.HttpScopes; -import com.passus.st.client.http.filter.HttpFilter; -import com.passus.st.client.http.filter.HttpFlowUtils; -import com.passus.st.emitter.SessionInfo; -import com.passus.st.source.EventDestination; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * @author mikolaj.podbielski - */ -class ConverterHttpClient implements Service, EventHandler, FilterAware { - - private static final Logger LOGGER = LogManager.getLogger(ConverterHttpClient.class); - - private final EventDestination dst; - private final FlowFilterChain filterChain = new FlowFilterChain(); - private final HttpScopes scopes = new HttpScopes(); - protected final Map<SessionInfo, FlowContext> sessions = new HashMap<>(); - - private boolean started; - - public ConverterHttpClient(EventDestination dst) { - this.dst = dst; - } - - @Override - public boolean isStarted() { - return started; - } - - @Override - public void start() { - if (started) { - return; - } - - dst.start(); - - started = true; - } - - @Override - public void stop() { - if (!started) { - return; - } - - dst.stop(); - - started = false; - } - - @Override - public void handle(Event event) { - if (!started) { - throw new IllegalArgumentException("Service not started."); - } - - if (LOGGER.isTraceEnabled()) { - LOGGER.trace("Event: {}", event); - } - - try { - switch (event.getType()) { - case SessionStatusEvent.TYPE: { - SessionStatusEvent e = (SessionStatusEvent) event; - // tworzyc HttpFlowContext ??? - if (e.getStatus() == SessionStatusEvent.STATUS_CLOSED) { - deregisterFlow(e.getSessionInfo()); - } - - break; - } - case SessionPayloadEvent.TYPE: { - SessionPayloadEvent e = (SessionPayloadEvent) event; - FlowContext flow = getFlow(e.getSessionInfo()); - int result = filterChain.filterOutbound(e.getRequest(), e.getResponse(), flow); - if (result == HttpFilter.ACCEPT) { - // krok potrzebny dla kilku filtrów, np session, digestLogin - flow.sentEvent(e); - filterChain.filterInbound(e.getRequest(), e.getResponse(), flow); - } else { - return; // skip dst.handle() - } - break; - } - default: - break; - } - - dst.handle(event); - } catch (Exception e) { - LOGGER.debug(e.getMessage(), e); - } - } - - @Override - public List<FlowFilter> getFilters() { - return filterChain.getFilters(); - } - - @Override - public void addFilter(FlowFilter filter) { - Assert.notNull(filter, "filter"); - filterChain.addFilter(filter); - } - - @Override - public void setFilters(Collection<FlowFilter> filters) { - Assert.notContainsNull(filters, "filters"); - filterChain.clear(); - filters.forEach(filterChain::addFilter); - } - - public void join() { - } - - HttpScopes scopes() { - return scopes; - } - - private FlowContext getFlow(SessionInfo session) { - FlowContext flowContext = sessions.get(session); - if (flowContext == null) { - flowContext = HttpFlowUtils.createFlowContext(session, scopes); - sessions.put(session, flowContext); - } - return flowContext; - } - - private FlowContext deregisterFlow(SessionInfo session) { - return sessions.remove(session); - } -}
--- a/stress-tester/src/main/java/com/passus/st/ConverterMain.java Tue Jun 16 16:53:08 2020 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,170 +0,0 @@ -package com.passus.st; - -import com.passus.config.ConfigurationContext; -import com.passus.st.filter.FlowFilter; -import com.passus.st.reader.nc.NcHttpWriteMode; -import com.passus.st.source.NcEventDestination; -import com.passus.st.source.PcapSessionEventSource; -import static com.passus.st.utils.CliUtils.option; -import java.io.File; -import java.util.List; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.DefaultParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; - -/** - * - * @author mikolaj.podbielski - */ -public class ConverterMain { - - private final CliHelper cliHelper = new CliHelper(); - - private static final String WRITE_MODES = "skip|headers-only|full-message"; - - private static NcHttpWriteMode resolveWriteMode(String raw, NcHttpWriteMode defaultValue) { - if (raw == null) { - return defaultValue; - } - - raw = raw.toLowerCase(); - switch (raw) { - case "skip": - return NcHttpWriteMode.SKIP; - case "headers-only": - return NcHttpWriteMode.HEADERS_ONLY; - case "full-message": - return NcHttpWriteMode.FULL_MESSAGE; - default: - throw new IllegalArgumentException("Invalid write mode '" + raw + "'."); - } - } - - private boolean askOverwriteIfExists(String output) { - File f = new File(output); - if (f.exists()) { - System.out.format("Destination file '%s' already exists. Overwrite? [y]/n\n", f.getAbsolutePath()); - String line = System.console().readLine(); - char yn = line.length() > 0 ? line.charAt(0) : 'y'; - return yn == 'y' || yn == 'Y'; - } - return true; - } - - static void printHelp(Options options) { - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("[options] <input pcap> <output nc>", "description", options, ""); - } - - private Options createOptions() { - final CliOptions options = cliHelper.options(); - - options.addLogLevelOption(); - options.addAllowPartialSessionOption(); - options.addLookupsFileOptions(); - options.addHttpFiltersFileOption(); - options.addVarsFileOption(); - options.addHttpPortsOption(); - - options.addOption(option("do", "").desc("Destination override ([ask]|yes|no)") - .hasArg().argName("") - .build() - ); - -// options.addOption(option("wm", "writeMetrics").desc("Write metrics to file.") -// .hasArg().argName("file") -// .build() -// ); - options.addOption(option("hreqpwm", "http-request-write-mode").desc(WRITE_MODES + " (default full-message)") - .hasArg().argName("mode") - .build() - ); - - options.addOption(option("hrespwm", "http-response-write-mode").desc(WRITE_MODES + " (default headers-only)") - .hasArg().argName("mode") - .build() - ); - - return options; - } - - public void start(String... args) { - AppUtils.registerAll(); - Options options = createOptions(); - - try { - CommandLine cl = new DefaultParser().parse(options, args); - String[] clArgs = cl.getArgs(); - if (clArgs.length < 2) { - System.err.println("At least one pcap file required."); - printHelp(options); - return; - } - String input = clArgs[0]; - String output = clArgs[1]; - System.out.println("Converting " + input + " to " + output); - - cliHelper.configureLogger(cl); - - boolean overwrite; - String overwriteOption = cl.getOptionValue("do", "ask").toLowerCase(); - switch (overwriteOption) { - case "yes": - overwrite = true; - break; - case "ask": - overwrite = askOverwriteIfExists(output); - break; - case "no": - overwrite = false; - break; - default: - throw new ParseException("Invalid override option value: " + overwriteOption); - } - - NcEventDestination dst = new NcEventDestination(output); - dst.setAllowOverwrite(overwrite); -/* - dst.setHttpRequestWriteMode(resolveWriteMode(cl.getOptionValue("hreqpwm"), NcHttpWriteMode.FULL_MESSAGE)); - dst.setHttpResponseWriteMode(resolveWriteMode(cl.getOptionValue("hrespwm"), NcHttpWriteMode.HEADERS_ONLY)); -*/ - - ConverterHttpClient client = new ConverterHttpClient(dst); - - ConfigurationContext context = ConfigurationContext.create(); - - cliHelper.readLookupsToContext(cl, context); - - List<FlowFilter> flowFilters = cliHelper.readFilters(cl, context); - if (flowFilters != null) { - flowFilters.forEach(client::addFilter); - } - - client.start(); - - PcapSessionEventSource eventSrc = cliHelper.createPcapEventSource(input, cl); - eventSrc.setHandler(client); - eventSrc.start(); - - eventSrc.join(); - client.join(); - eventSrc.stop(); - client.stop(); - cliHelper.beforeExit(context); - } catch (ParseException e) { - System.out.println(e.getMessage()); - printHelp(options); - } catch (Exception e) { - e.printStackTrace(System.err); - } finally { - AppUtils.unregisterAll(); - } - } - - public static void main(String[] args) { - new ConverterMain().start(args); - } - -}
--- a/stress-tester/src/main/java/com/passus/st/GlobalConfigMain.java Tue Jun 16 16:53:08 2020 +0200 +++ b/stress-tester/src/main/java/com/passus/st/GlobalConfigMain.java Wed Jun 17 11:23:46 2020 +0200 @@ -5,7 +5,7 @@ import com.passus.config.ConfigurationContext; import com.passus.config.YamlConfigurationReader; import com.passus.config.validation.Errors; -import com.passus.st.config.TestJobConfigurator; +import com.passus.st.config.JobConfigurator; import com.passus.st.job.TestJob; import com.passus.st.metric.MetricsCollector; import com.passus.st.metric.SummaryMetricsCollectionHandler; @@ -66,8 +66,8 @@ ConfigurationContext context = ConfigurationContext.create(); - TestJobConfigurator testJobConfigurator = new TestJobConfigurator(); - testJobConfigurator.configure(config, errors, context); + JobConfigurator jobConfigurator = new JobConfigurator(); + jobConfigurator.configure(config, errors, context); if (errors.hasError()) { StringBuilder sb = new StringBuilder(); @@ -109,7 +109,7 @@ } testJob.stop(); - testJobConfigurator.destroy(config, errors, context); + jobConfigurator.destroy(config, errors, context); logger.debug("TestJob stopped"); } catch (ParseException e) { System.out.println(e.getMessage());
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/stress-tester/src/main/java/com/passus/st/config/JobConfigurator.java Wed Jun 17 11:23:46 2020 +0200 @@ -0,0 +1,143 @@ +package com.passus.st.config; + +import com.passus.commons.Assert; +import com.passus.commons.plugin.PluginFactory; +import com.passus.config.*; +import com.passus.config.validation.Errors; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.function.BiPredicate; + +import static com.passus.config.ConfigurationUtils.validateType; + +/** + * @author Mirosław Hawrot + */ +public class JobConfigurator { + + private static final Logger LOGGER = LogManager.getLogger(JobConfigurator.class); + + private PluginFactory<DomainConfigurator> configuratorFactory = DomainConfiguratorFactory.getInstance(); + + private final BiPredicate<String, DomainConfigurator> filter; + + public JobConfigurator() { + this(null); + } + + protected JobConfigurator(BiPredicate<String, DomainConfigurator> filter) { + this.filter = filter; + } + + public PluginFactory<DomainConfigurator> getConfiguratorFactory() { + return configuratorFactory; + } + + public void setConfiguratorFactory(PluginFactory<DomainConfigurator> configuratorFactory) { + Assert.notNull(configuratorFactory, "configuratorFactory"); + this.configuratorFactory = configuratorFactory; + } + + protected Map<String, DomainConfigurator> getDomainsConfigurator(Configuration config, Errors errors) { + return getDomainsConfigurator(config.getRootNode(), errors); + } + + protected Map<String, DomainConfigurator> getDomainsConfigurator(CNode rootNode, Errors errors) { + if (!validateType(rootNode, NodeType.MAP, errors)) { + return null; + } + + CMapNode mapNode = (CMapNode) rootNode; + Map<String, DomainConfigurator> domainsConfigurator = new LinkedHashMap<>(); + List<CTupleNode> tuples = mapNode.getChildren(); + for (CTupleNode tuple : tuples) { + String domain = tuple.getName(); + if (filter != null && filter.test(domain, null)) { + continue; + } + + DomainConfigurator configurator = null; + + try { + configurator = configuratorFactory.getInstanceByName(domain); + } catch (IllegalArgumentException e) { + } + + if (configurator == null) { + errors.reject(tuple, "Unknown property '%s'.", domain); + break; + } + + domainsConfigurator.put(domain, configurator); + } + + return domainsConfigurator; + } + + public void validate(Configuration config, Errors errors, ConfigurationContext context) { + Map<String, DomainConfigurator> configurators = getDomainsConfigurator(config, errors); + if (!errors.hasError()) { + for (Entry<String, DomainConfigurator> entry : configurators.entrySet()) { + String keyName = entry.getKey(); + if (filter != null && filter.test(keyName, entry.getValue())) { + continue; + } + + Configuration subConfig = config.subConfiguration(keyName); + DomainConfigurator configurator = entry.getValue(); + configurator.validate(subConfig, errors, context); + if (errors.hasError()) { + break; + } + } + } + } + + public void configure(Configuration config, Errors errors, ConfigurationContext context) { + Map<String, DomainConfigurator> configurators = getDomainsConfigurator(config, errors); + if (!errors.hasError()) { + for (Entry<String, DomainConfigurator> entry : configurators.entrySet()) { + String keyName = entry.getKey(); + if (filter != null && filter.test(keyName, entry.getValue())) { + continue; + } + + Configuration subConfig = config.subConfiguration(keyName); + DomainConfigurator configurator = entry.getValue(); + configurator.validate(subConfig, errors, context); + if (errors.hasError()) { + break; + } + configurator.configure(subConfig, errors, context); + if (errors.hasError()) { + break; + } + configurator.afterConfigurationSet(errors, context); + if (errors.hasError()) { + break; + } + } + } + } + + public void destroy(Configuration config, Errors errors, ConfigurationContext context) { + Map<String, DomainConfigurator> configurators = getDomainsConfigurator(config, errors); + if (!errors.hasError()) { + for (Entry<String, DomainConfigurator> entry : configurators.entrySet()) { + String keyName = entry.getKey(); + if (filter != null && filter.test(keyName, entry.getValue())) { + continue; + } + + Configuration subConfig = config.subConfiguration(keyName); + DomainConfigurator configurator = entry.getValue(); + configurator.destroy(subConfig, context); + } + } + } +}
--- a/stress-tester/src/main/java/com/passus/st/config/TestJobConfigurator.java Tue Jun 16 16:53:08 2020 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,143 +0,0 @@ -package com.passus.st.config; - -import com.passus.commons.Assert; -import com.passus.commons.plugin.PluginFactory; -import com.passus.config.*; -import com.passus.config.validation.Errors; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.function.BiPredicate; - -import static com.passus.config.ConfigurationUtils.validateType; - -/** - * @author Mirosław Hawrot - */ -public class TestJobConfigurator { - - private static final Logger LOGGER = LogManager.getLogger(TestJobConfigurator.class); - - private PluginFactory<DomainConfigurator> configuratorFactory = DomainConfiguratorFactory.getInstance(); - - private final BiPredicate<String, DomainConfigurator> filter; - - public TestJobConfigurator() { - this(null); - } - - protected TestJobConfigurator(BiPredicate<String, DomainConfigurator> filter) { - this.filter = filter; - } - - public PluginFactory<DomainConfigurator> getConfiguratorFactory() { - return configuratorFactory; - } - - public void setConfiguratorFactory(PluginFactory<DomainConfigurator> configuratorFactory) { - Assert.notNull(configuratorFactory, "configuratorFactory"); - this.configuratorFactory = configuratorFactory; - } - - protected Map<String, DomainConfigurator> getDomainsConfigurator(Configuration config, Errors errors) { - return getDomainsConfigurator(config.getRootNode(), errors); - } - - protected Map<String, DomainConfigurator> getDomainsConfigurator(CNode rootNode, Errors errors) { - if (!validateType(rootNode, NodeType.MAP, errors)) { - return null; - } - - CMapNode mapNode = (CMapNode) rootNode; - Map<String, DomainConfigurator> domainsConfigurator = new LinkedHashMap<>(); - List<CTupleNode> tuples = mapNode.getChildren(); - for (CTupleNode tuple : tuples) { - String domain = tuple.getName(); - if (filter != null && filter.test(domain, null)) { - continue; - } - - DomainConfigurator configurator = null; - - try { - configurator = configuratorFactory.getInstanceByName(domain); - } catch (IllegalArgumentException e) { - } - - if (configurator == null) { - errors.reject(tuple, "Unknown property '%s'.", domain); - break; - } - - domainsConfigurator.put(domain, configurator); - } - - return domainsConfigurator; - } - - public void validate(Configuration config, Errors errors, ConfigurationContext context) { - Map<String, DomainConfigurator> configurators = getDomainsConfigurator(config, errors); - if (!errors.hasError()) { - for (Entry<String, DomainConfigurator> entry : configurators.entrySet()) { - String keyName = entry.getKey(); - if (filter != null && filter.test(keyName, entry.getValue())) { - continue; - } - - Configuration subConfig = config.subConfiguration(keyName); - DomainConfigurator configurator = entry.getValue(); - configurator.validate(subConfig, errors, context); - if (errors.hasError()) { - break; - } - } - } - } - - public void configure(Configuration config, Errors errors, ConfigurationContext context) { - Map<String, DomainConfigurator> configurators = getDomainsConfigurator(config, errors); - if (!errors.hasError()) { - for (Entry<String, DomainConfigurator> entry : configurators.entrySet()) { - String keyName = entry.getKey(); - if (filter != null && filter.test(keyName, entry.getValue())) { - continue; - } - - Configuration subConfig = config.subConfiguration(keyName); - DomainConfigurator configurator = entry.getValue(); - configurator.validate(subConfig, errors, context); - if (errors.hasError()) { - break; - } - configurator.configure(subConfig, errors, context); - if (errors.hasError()) { - break; - } - configurator.afterConfigurationSet(errors, context); - if (errors.hasError()) { - break; - } - } - } - } - - public void destroy(Configuration config, Errors errors, ConfigurationContext context) { - Map<String, DomainConfigurator> configurators = getDomainsConfigurator(config, errors); - if (!errors.hasError()) { - for (Entry<String, DomainConfigurator> entry : configurators.entrySet()) { - String keyName = entry.getKey(); - if (filter != null && filter.test(keyName, entry.getValue())) { - continue; - } - - Configuration subConfig = config.subConfiguration(keyName); - DomainConfigurator configurator = entry.getValue(); - configurator.destroy(subConfig, context); - } - } - } -}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/stress-tester/src/main/java/com/passus/st/converter/Converter.java Wed Jun 17 11:23:46 2020 +0200 @@ -0,0 +1,150 @@ +package com.passus.st.converter; + +import com.passus.config.Configuration; +import com.passus.config.ConfigurationContext; +import com.passus.config.validation.Errors; +import com.passus.st.AppUtils; +import com.passus.st.CliHelper; +import com.passus.st.CliOptions; +import com.passus.st.config.JobConfigurator; +import com.passus.st.reader.nc.NcHttpWriteMode; +import com.passus.st.source.NcEventDestination; +import com.passus.st.source.PcapSessionEventSource; +import org.apache.commons.cli.*; + +import java.io.File; + +import static com.passus.st.utils.CliUtils.option; + +/** + * @author mikolaj.podbielski + */ +public class Converter { + + private final CliHelper cliHelper = new CliHelper(); + + private static final String WRITE_MODES = "skip|headers-only|full-message"; + + private static NcHttpWriteMode resolveWriteMode(String raw, NcHttpWriteMode defaultValue) { + if (raw == null) { + return defaultValue; + } + + raw = raw.toLowerCase(); + switch (raw) { + case "skip": + return NcHttpWriteMode.SKIP; + case "headers-only": + return NcHttpWriteMode.HEADERS_ONLY; + case "full-message": + return NcHttpWriteMode.FULL_MESSAGE; + default: + throw new IllegalArgumentException("Invalid write mode '" + raw + "'."); + } + } + + private boolean askOverwriteIfExists(String output) { + File f = new File(output); + if (f.exists()) { + System.out.format("Destination file '%s' already exists. Overwrite? [y]/n\n", f.getAbsolutePath()); + String line = System.console().readLine(); + char yn = line.length() > 0 ? line.charAt(0) : 'y'; + return yn == 'y' || yn == 'Y'; + } + return true; + } + + static void printHelp(Options options) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("[options] <output nc>", "description", options, ""); + } + + private Options createOptions() { + final CliOptions options = cliHelper.options(); + + options.addLogLevelOption(); + options.addAllowPartialSessionOption(); + + options.addOption(option("do", "").desc("Destination override ([ask]|yes|no)") + .hasArg().argName("") + .build() + ); + + options.addOption(option("c", "config-file").desc("Configuration file.") + .hasArg().argName("file") + .build() + ); + + return options; + } + + private boolean processOverwriteOption(CommandLine cl, String output) throws ParseException { + String overwriteOption = cl.getOptionValue("do", "ask").toLowerCase(); + switch (overwriteOption) { + case "yes": + return true; + case "ask": + return askOverwriteIfExists(output); + case "no": + return false; + default: + throw new ParseException("Invalid override option value: " + overwriteOption); + } + } + + public void start(String... args) { + AppUtils.registerAll(); + Options options = createOptions(); + + try { + CommandLine cl = new DefaultParser().parse(options, args); + String[] clArgs = cl.getArgs(); + if (clArgs.length != 1) { + System.err.println("Output file required."); + printHelp(options); + return; + } + + String output = clArgs[0]; + System.out.println("Converting to " + output); + cliHelper.configureLogger(cl); + + boolean overwrite = processOverwriteOption(cl, output); + + ConfigurationContext context = ConfigurationContext.create(); + Errors errors = new Errors(); + JobConfigurator jobConfigurator = null; + Configuration config = null; + if (cl.hasOption("c")) { + String configFile = cl.getOptionValue("c"); + ConverterConfiguratorPluginFactory pluginFactory = new ConverterConfiguratorPluginFactory(); + config = cliHelper.readConfiguration(configFile); + jobConfigurator = cliHelper.processJobConfiguration(config, context, errors, pluginFactory); + } + + NcEventDestination dst = new NcEventDestination(output); + dst.setAllowOverwrite(overwrite); + + ConverterJob converterJob = ConverterJob.create(context, dst); + converterJob.start(); + converterJob.join(); + converterJob.stop(); + + if (jobConfigurator != null) { + jobConfigurator.destroy(config, errors, context); + } + } catch (ParseException e) { + System.out.println(e.getMessage()); + printHelp(options); + } catch (Exception e) { + e.printStackTrace(System.err); + } finally { + AppUtils.unregisterAll(); + } + } + + public static void main(String[] args) { + new Converter().start(args); + } + +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/stress-tester/src/main/java/com/passus/st/converter/ConverterConfiguratorPluginFactory.java Wed Jun 17 11:23:46 2020 +0200 @@ -0,0 +1,19 @@ +package com.passus.st.converter; + +import com.passus.commons.plugin.PluginFactory; +import com.passus.config.DomainConfigurator; +import com.passus.st.filter.FlowFiltersConfigurator; +import com.passus.st.lookup.LookupsDomainConfigurator; +import com.passus.st.source.EventSourceConfigurator; +import com.passus.st.vars.VarsGlobalDomainConfigurator; + +public class ConverterConfiguratorPluginFactory extends PluginFactory<DomainConfigurator> { + + public ConverterConfiguratorPluginFactory() { + add(EventSourceConfigurator.DOMAIN, EventSourceConfigurator.class); + add(LookupsDomainConfigurator.DOMAIN, LookupsDomainConfigurator.class); + add(FlowFiltersConfigurator.DOMAIN, FlowFiltersConfigurator.class); + add(VarsGlobalDomainConfigurator.DOMAIN, VarsGlobalDomainConfigurator.class); + } + +}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/stress-tester/src/main/java/com/passus/st/converter/ConverterJob.java Wed Jun 17 11:23:46 2020 +0200 @@ -0,0 +1,167 @@ +package com.passus.st.converter; + +import com.passus.commons.Assert; +import com.passus.config.ConfigurationContext; +import com.passus.st.client.*; +import com.passus.st.client.http.HttpScopes; +import com.passus.st.client.http.filter.HttpFilter; +import com.passus.st.client.http.filter.HttpFlowUtils; +import com.passus.st.emitter.SessionInfo; +import com.passus.st.filter.FlowFilter; +import com.passus.st.filter.FlowFilterChain; +import com.passus.st.job.Job; +import com.passus.st.source.EventDestination; +import com.passus.st.source.EventSource; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static com.passus.st.job.TestJob.populateIfNotNull; +import static com.passus.st.utils.ConfigurationContextConsts.EVENT_SOURCE_SOURCES; +import static com.passus.st.utils.ConfigurationContextConsts.FLOW_FILTERS; + +/** + * @author mikolaj.podbielski + */ +class ConverterJob implements Job { + + private static final Logger LOGGER = LogManager.getLogger(ConverterJob.class); + + private List<EventSource> eventSources = new ArrayList<>(); + + final FlowFilterChain filterChain = new FlowFilterChain(); + + private final HttpScopes scopes = new HttpScopes(); + + protected final Map<SessionInfo, FlowContext> sessions = new HashMap<>(); + + private final EventDestination destination; + + private boolean started; + + private boolean wait = false; + + protected FlowHandlerFactory clientFactory = new FlowHandlerFactoryImpl(); + + public ConverterJob(EventDestination destination) { + this.destination = destination; + } + + @Override + public void start() { + if (started) { + return; + } + + destination.start(); + eventSources.forEach(src -> { + src.setHandler(this::handle); + src.start(); + }); + + wait = true; + started = true; + } + + @Override + public void stop() { + if (!started) { + return; + } + + eventSources.forEach(EventSource::stop); + destination.stop(); + started = false; + } + + public void join() { + while (wait) { + try { + Thread.sleep(100); + } catch (InterruptedException ignore) { + + } + } + } + + void handle(Event event) { + if (LOGGER.isTraceEnabled()) { + LOGGER.trace("Event: {}", event); + } + + try { + switch (event.getType()) { + case SessionStatusEvent.TYPE: { + SessionStatusEvent e = (SessionStatusEvent) event; + // tworzyc HttpFlowContext ??? + if (e.getStatus() == SessionStatusEvent.STATUS_CLOSED) { + deregisterFlow(e.getSessionInfo()); + } + + break; + } + case SessionPayloadEvent.TYPE: { + SessionPayloadEvent e = (SessionPayloadEvent) event; + FlowContext flow = getFlow(e.getSessionInfo()); + int result = filterChain.filterOutbound(e.getRequest(), e.getResponse(), flow); + if (result == HttpFilter.ACCEPT) { + // krok potrzebny dla kilku filtrów, np session, digestLogin + flow.sentEvent(e); + filterChain.filterInbound(e.getRequest(), e.getResponse(), flow); + } else { + return; // skip dst.handle() + } + break; + } + case DataEvents.DataEnd.TYPE: { + wait = false; + } + default: + break; + } + + destination.handle(event); + } catch (Exception e) { + LOGGER.debug(e.getMessage(), e); + } + } + + HttpScopes scopes() { + return scopes; + } + + FlowContext getFlow(SessionInfo session) { + FlowContext flowContext = sessions.get(session); + if (flowContext == null) { + flowContext = HttpFlowUtils.createFlowContext(session, scopes); + FlowHandler flowHandler = clientFactory.create(session.getProtocolId()); + flowHandler.init(flowContext); + sessions.put(session, flowContext); + } + return flowContext; + } + + private FlowContext deregisterFlow(SessionInfo session) { + return sessions.remove(session); + } + + public static ConverterJob create(ConfigurationContext context, EventDestination dst) { + Assert.notNull(context, "context"); + Assert.notNull(dst, "dst"); + ConverterJob job = new ConverterJob(dst); + + List<EventSource> cfgEventSources = context.get(EVENT_SOURCE_SOURCES); + populateIfNotNull(cfgEventSources, job.eventSources); + + List<FlowFilter> flowFilters = context.get(FLOW_FILTERS); + if (flowFilters != null) { + flowFilters.forEach(job.filterChain::addFilter); + } + + return job; + } +}
--- a/stress-tester/src/main/java/com/passus/st/job/TestJob.java Tue Jun 16 16:53:08 2020 +0200 +++ b/stress-tester/src/main/java/com/passus/st/job/TestJob.java Wed Jun 17 11:23:46 2020 +0200 @@ -126,7 +126,7 @@ flowExecutor.join(); } - private static <T> void populateIfNotNull(Collection<T> srcCollection, Collection<T> dstCollection) { + public static <T> void populateIfNotNull(Collection<T> srcCollection, Collection<T> dstCollection) { if (srcCollection != null) { dstCollection.addAll(srcCollection); }
--- a/stress-tester/src/main/java/com/passus/st/job/TestJobBuilder.java Tue Jun 16 16:53:08 2020 +0200 +++ b/stress-tester/src/main/java/com/passus/st/job/TestJobBuilder.java Wed Jun 17 11:23:46 2020 +0200 @@ -8,7 +8,7 @@ import com.passus.st.client.FlowExecutor; import com.passus.st.client.MultipleClientListener; import com.passus.st.client.http.ReporterDestination; -import com.passus.st.config.TestJobConfigurator; +import com.passus.st.config.JobConfigurator; import com.passus.st.emitter.Emitter; import com.passus.st.emitter.nio.NioEmitter; import com.passus.st.metric.MetricsCollector; @@ -146,7 +146,7 @@ } private void process(Configuration config, Errors errors) { - TestJobConfigurator configurator = new TestJobConfigurator(); + JobConfigurator configurator = new JobConfigurator(); ConfigurationContext context = ConfigurationContext.create(); configurator.configure(config, errors, context); if (!errors.hasError()) {
--- a/stress-tester/src/main/java/com/passus/st/project/ProjectConfigurator.java Tue Jun 16 16:53:08 2020 +0200 +++ b/stress-tester/src/main/java/com/passus/st/project/ProjectConfigurator.java Wed Jun 17 11:23:46 2020 +0200 @@ -4,7 +4,7 @@ import com.passus.config.ConfigurationContext; import com.passus.config.DomainConfigurator; import com.passus.config.validation.Errors; -import com.passus.st.config.TestJobConfigurator; +import com.passus.st.config.JobConfigurator; import com.passus.st.utils.ConfigurationContextConsts; import java.util.Arrays; @@ -14,7 +14,7 @@ import static com.passus.config.schema.validation.NodeValidationMessages.TUPLE_NOT_DEFINED_MSG; -public class ProjectConfigurator extends TestJobConfigurator { +public class ProjectConfigurator extends JobConfigurator { private static final Set<String> PROJECT_KEYS = new HashSet<>( Arrays.asList("name", "active", "description")
--- a/stress-tester/src/main/java/com/passus/st/scanner/Scanner.java Tue Jun 16 16:53:08 2020 +0200 +++ b/stress-tester/src/main/java/com/passus/st/scanner/Scanner.java Wed Jun 17 11:23:46 2020 +0200 @@ -8,7 +8,7 @@ import com.passus.st.AppUtils; import com.passus.st.CliHelper; import com.passus.st.CliOptions; -import com.passus.st.config.TestJobConfigurator; +import com.passus.st.config.JobConfigurator; import com.passus.st.metric.MetricsCollector; import com.passus.st.metric.SummaryMetricsCollectionHandler; import org.apache.commons.cli.*; @@ -58,29 +58,14 @@ cliHelper.configureLogger(cl); final String metricFormat = cl.getOptionValue("mf", "text"); - Errors errors = new Errors(); - File configFile = new File(clArgs[0]); - Configuration config = YamlConfigurationReader.readFromFile(configFile); - ConfigurationContext context = ConfigurationContext.create(); ScannerConfiguratorPluginFactory pluginFactory = new ScannerConfiguratorPluginFactory(); - TestJobConfigurator testJobConfigurator = new TestJobConfigurator(); - testJobConfigurator.setConfiguratorFactory(pluginFactory); - - testJobConfigurator.configure(config, errors, context); - if (errors.hasError()) { - StringBuilder sb = new StringBuilder(); - - errors.getAllErrors().forEach(error -> { - sb.append(error.toString()).append("\n"); - }); - - cliHelper.printError(sb.toString()); - } + Errors errors = new Errors(); + Configuration config = cliHelper.readConfiguration(clArgs[0]); + JobConfigurator jobConfigurator = cliHelper.processJobConfiguration(config, context, errors, pluginFactory); ScannerJob job = ScannerJob.create(context); - final long startTime = System.currentTimeMillis(); MetricsCollector metricsCollector = job.getMetricsCollector(); @@ -90,7 +75,7 @@ job.start(); job.stop(); - testJobConfigurator.destroy(config, errors, context); + jobConfigurator.destroy(config, errors, context); printMetrics(getMetrics(), startTime, metricFormat); logger.debug("Scanner stopped");
--- a/stress-tester/src/main/java/com/passus/st/source/PcapHttpSessionAnalyzerHook.java Tue Jun 16 16:53:08 2020 +0200 +++ b/stress-tester/src/main/java/com/passus/st/source/PcapHttpSessionAnalyzerHook.java Wed Jun 17 11:23:46 2020 +0200 @@ -1,6 +1,7 @@ package com.passus.st.source; import com.passus.net.http.session.HttpSessionAnalyzer; +import com.passus.net.http2.session.Http2SessionAnalyzer; import com.passus.net.session.SessionAnalyzer; import com.passus.net.session.TcpSessionProcessor; @@ -8,7 +9,8 @@ @Override public boolean supports(Class<? extends SessionAnalyzer> clazz) { - return HttpSessionAnalyzer.class.isAssignableFrom(clazz); + return HttpSessionAnalyzer.class.isAssignableFrom(clazz) + || Http2SessionAnalyzer.class.isAssignableFrom(clazz); } @Override
--- a/stress-tester/src/test/java/com/passus/st/ConverterHttpClientTest.java Tue Jun 16 16:53:08 2020 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,294 +0,0 @@ -package com.passus.st; - -import com.passus.net.http.*; -import com.passus.st.client.SessionPayloadEvent; -import com.passus.st.client.credentials.Credentials; -import com.passus.st.client.credentials.CredentialsProvider; -import com.passus.st.client.http.filter.*; -import com.passus.st.client.http.filter.HttpMessageModificationOperations.PostDataSetParamOperation; -import com.passus.st.client.http.filter.HttpMessageModificationOperations.SetCookieOperation; -import com.passus.st.client.http.filter.HttpMessageModificationOperations.SetHeaderOperation; -import com.passus.st.client.http.filter.HttpMessageModificationOperations.SetQueryParameterOperation; -import com.passus.st.emitter.SessionInfo; -import com.passus.st.filter.*; -import com.passus.st.source.ArrayListEventDestination; -import com.passus.st.source.NullEventDestination; -import com.passus.st.utils.TestHttpUtils; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; - -import java.io.IOException; -import java.util.List; - -import static com.passus.commons.collection.FluentBuilder.e; -import static com.passus.commons.collection.FluentBuilder.map; -import static com.passus.st.Protocols.HTTP; -import static com.passus.st.client.http.HttpConsts.*; -import static org.testng.AssertJUnit.*; - -/** - * @author mikolaj.podbielski - */ -public class ConverterHttpClientTest { - - static SessionInfo SI = new SessionInfo("1.1.1.1", 11000, "1.1.1.2", 80); - static Credentials CREDENTIALS = new Credentials("test", "test"); - static CredentialsProvider PROVIDER = HttpAbstractLoginFilterTest.provider(CREDENTIALS); - static NullEventDestination DST = new NullEventDestination(); - - static { - //Log4jConfigurationFactory.enableFactory(Level.DEBUG); - } - - @BeforeClass - public static void beforeClass() { - AppUtils.registerAll(); - } - - @AfterClass - public static void afterClass() { - AppUtils.unregisterAll(); - } - - static SessionPayloadEvent ev(HttpRequest req, HttpResponse resp) { - return new SessionPayloadEvent(SI, req, resp, HTTP, "test"); - } - - static ConverterHttpClient client(FlowFilter... filters) { - ConverterHttpClient client = new ConverterHttpClient(DST); - for (FlowFilter filter : filters) { - client.addFilter(filter); - } - client.start(); - return client; - } - - @Test - public void testBasicLogin_filterShouldReplaceCredentials() { - final String REQS = "GET /basic/index.html HTTP/1.1\r\n" - + "Authorization: Basic dXNlcjpwYXNzd29yZA==\r\n\r\n"; - HttpRequest req = TestHttpUtils.request(REQS); - HttpResponse resp = HttpResponseBuilder.ok().build(); - - HttpBasicAuthLoginFilter f = HttpBasicAuthLoginFilterTest.createFilter(PROVIDER); - ConverterHttpClient client = client(f); - client.handle(ev(req, resp)); - - assertEquals(req.getHeaders().get(HttpHeaders.AUTHORIZATION).toString(), "Basic dGVzdDp0ZXN0"); - assertEquals(client.scopes().getConversation(req).get(PARAM_USERNAME), "test"); - } - - @Test - public void testDigestLogin_filterShouldReplaceCredentials() { - final String RESP1S = "HTTP/1.1 401 Unauthorized\r\n" - + "Date: Fri, 09 Jun 2017 06:25:27 GMT\r\n" - + "WWW-Authenticate: Digest realm=\"realm\", nonce=\"old-nonce\", algorithm=MD5, qop=\"auth\"\r\n" - + "Content-Length: 12\r\n\r\nUnauthorized"; - final String REQ2S = "GET /html/digest/index2.html HTTP/1.1\r\n" - + "Host: 172.16.60.101\r\n" - + "Authorization: Digest username=\"user\", realm=\"realm\", nonce=\"old-nonce\"," - + " uri=\"/html/digest/index2.html\", algorithm=MD5, response=\"774280b22c9ed40bc500bba5c431d8c7\"," - + " qop=auth, nc=00000001, cnonce=\"cnonce\"\r\n\r\n"; - final String RESP2S = "HTTP/1.1 200 OK\n" - + "Authentication-Info: rspauth=\"?\", cnonce=\"?\", nc=00000001, qop=auth\r\n" - + "Content-Length: 33\r\n\r\n<html><body>content</body></html>"; - - HttpRequest req1 = HttpRequestBuilder.get("http://html/digest/index2.html").build(); - HttpResponse resp1 = TestHttpUtils.response(RESP1S); - HttpRequest req2 = TestHttpUtils.request(REQ2S); - HttpResponse resp2 = TestHttpUtils.response(RESP2S); - req2.setTag(TAG_SESSION_ID, "sid1"); - - HttpDigestAuthLoginFilter f = HttpDigestAuthLoginFilterTest.createFilter(CREDENTIALS); - ConverterHttpClient client = client(f); - client.handle(ev(req1, resp1)); - client.handle(ev(req2, resp2)); - - assertEquals(client.scopes().getConversation(req2).get(PARAM_USERNAME), "test"); - assertEquals(client.scopes().getSession(req2).get(PARAM_USERNAME), "test"); - } - - @Test - public void testFormLogin() throws IOException { - final String REQS = "POST /login_check HTTP/1.1\r\n" - + "Host: 172.16.60.23\r\n" - + "Content-Type: application/x-www-form-urlencoded\r\n" - + "Content-Length: 32\r\n" - + "Cookie: PHPSESSID=4r7tg8275ih7p0uh79im6q5643\r\n\r\n" - + "_username=admin&_password=qwerty"; - final String RESPS = "HTTP/1.1 302 Found\r\n" - + "Set-Cookie: PHPSESSID=pcap_post; path=/; HttpOnly\r\n" - + "Location: http://172.16.60.23/\r\n" - + "Content-Length: 2\r\n\r\n" - + "OK"; - - HttpRequest req = TestHttpUtils.request(REQS); - HttpResponse resp = TestHttpUtils.response(RESPS); - req.setTag(TAG_SESSION_ID, "sid1"); - - HttpFormLoginFilter f = HttpFormLoginFilterTest.createFilter(PROVIDER); - ConverterHttpClient client = client(f); - client.handle(ev(req, resp)); - - HttpFormLoginFilterTest.assertContent(req, "_username=test&_password=test"); - assertEquals(client.scopes().getConversation(req).get(PARAM_USERNAME), "test"); - assertEquals(client.scopes().getSession(req).get(PARAM_USERNAME), "test"); - } - - @Test - public void testMatcher() { - HttpRequest req1 = HttpRequestBuilder.get("http://test.com/test").build(); - HttpRequest req2 = HttpRequestBuilder.get("http://test2.com/test2").build(); - HttpResponse resp1 = HttpResponseBuilder.ok().build(); - HttpResponse resp2 = HttpResponseBuilder.status(HttpStatus.NO_CONTENT).build(); - - ArrayListEventDestination dst = new ArrayListEventDestination(); - MessagePredicate predicate = FilterTestUtils.createPredicate("{'@req.url': {$contains: test2}}"); - MatchFilter f = new MatchFilter(); - f.setPredicate(predicate); - f.setAcceptOnMatch(false); - - ConverterHttpClient client = new ConverterHttpClient(dst); - client.addFilter(f); - client.start(); - - client.handle(ev(req1, resp1)); // match:false filter:dunno chain:accept - client.handle(ev(req2, resp2)); // match:true filter:deny chain:deny - - assertEquals(1, dst.getEvents().size()); - SessionPayloadEvent event = dst.findFirst(SessionPayloadEvent.class); - assertEquals(200, ((HttpResponse) event.getResponse()).getStatus().getCode()); - } - - @Test - public void testModifyMessage() throws IOException { - HttpRequest req = HttpRequestBuilder - .post("http://test.com/path/test?uParam1=1", "param1=value1¶m2=value2") - .header("Header1", "Header1Value1") - .header("Cookie", "myCookie1=myValue1;myCookie2=myValue2") - .build(); - - HttpMessageModificationFilter f = new HttpMessageModificationFilter(); - f.addOperation(new SetHeaderOperation("Header1", "Header1Value1a")); - f.addOperation(new SetCookieOperation("myCookie1", "myValue1a")); - f.addOperation(new PostDataSetParamOperation("param1", "value1a")); - f.addOperation(new SetQueryParameterOperation("uParam1", "11")); - ConverterHttpClient client = client(f); - - client.handle(ev(req, null)); - HttpHeaders headers = req.getHeaders(); - HttpParameters params = HttpMessageHelper.get().decodeFormUrlencoded(req); - - assertEquals("/path/test?uParam1=11", req.getUri().toString()); - assertEquals("Header1Value1a", headers.get("Header1").toString()); - assertEquals("myCookie1=myValue1a;myCookie2=myValue2", headers.get("Cookie").toString()); - assertEquals("param2=value2¶m1=value1a", params.toString()); - } - - @Test - public void testDateRewriter_() { - HttpRequest req = HttpRequestBuilder.get("http://example.com/") - .header(HttpHeaders.DATE, "Wed, 09-Sep-2009 09:09:00 GMT") - .build(); - - HttpDateFilter f = new HttpDateFilter(); - HttpDateFilterTest.setDate(f, 1500_000_000_000L); - ConverterHttpClient client = client(f); - - client.handle(ev(req, null)); - assertEquals(req.getHeaders().get(HttpHeaders.DATE).toString(), "Fri, 14 Jul 2017 02:40:00 GMT"); - } - - @Test - public void testHostRewriter() { - HttpRequest req = HttpRequestBuilder.get("http://example.com:8080/site1/resource1") - .header(HttpHeaders.ORIGIN, "http://example.com:8080/site1") - .header(HttpHeaders.REFERER, "http://example.com:8080/site1/index") - .build(); - - HttpHostRewriterFilter f = new HttpHostRewriterFilter(); - f.setHostMap(map(e("example.com:8080", "example.org"))); - ConverterHttpClient client = client(f); - - client.handle(ev(req, null)); - - HttpHeaders headers = req.getHeaders(); - assertEquals(headers.get(HttpHeaders.HOST).toString(), "example.org"); - assertEquals(headers.get(HttpHeaders.ORIGIN).toString(), "http://example.org/site1"); - assertEquals(headers.get(HttpHeaders.REFERER).toString(), "http://example.org/site1/index"); - } - - @Test - public void testHeaderCleaners() { - HttpRequest req = HttpRequestBuilder.get("http://www.example.com/abc") - .header(HttpHeaders.ACCEPT, "*/*;q=0.5") - .header(HttpHeaders.IF_MODIFIED_SINCE, "09 Sep 2009 09:09:00 GMT") - .header(HttpHeaders.X_FORWARDED_FOR, "15.16.73.33") - .build(); - HttpResponse resp = HttpResponseBuilder.ok().build(); - - HttpFilter f1 = new HttpRequestCacheHeadersCleanerFilter(); - HttpFilter f2 = new HttpRequestProxyHeadersCleanerFilter(); - ConverterHttpClient client = client(f1, f2); - - client.handle(ev(req, resp)); - HttpHeaders headers = req.getHeaders(); - - assertTrue(headers.contains(HttpHeaders.ACCEPT)); - assertFalse(headers.contains(HttpHeaders.IF_MODIFIED_SINCE)); - assertFalse(headers.contains(HttpHeaders.X_FORWARDED_FOR)); - } - - @Test - public void testZone_() { - HttpRequest req = HttpRequestBuilder.get("http://test.com/test").build(); - HttpResponse resp = HttpResponseBuilder.ok().build(); - MessagePredicate predicate = FilterTestUtils.createPredicate("{\"@req.url\": {$contains: test}}"); - - HttpZoneFilter f = new HttpZoneFilter(); - f.addRule(new HttpZoneFilter.Rule("testZone", predicate)); - ConverterHttpClient client = client(f); - - client.handle(ev(req, resp)); - - assertEquals("testZone", req.getTag(TAG_ZONE)); - assertEquals("testZone", resp.getTag(TAG_ZONE)); - } - - @Test - public void testMVEL() { - HttpRequest req = HttpRequestBuilder.get("http://example.com/index.html").build(); - - String expression = "$req.setMethod(com.passus.net.http.HttpMethod.HEAD);" - + "$req.getUri().toString().equals(\"/index.html\") ? 1 : 0"; - - MvelFilter f = new MvelFilter(); - f.setStatement(MvelFilterTest.es(expression)); - f.setDirection(FilterDirection.OUT); - ConverterHttpClient client = client(f); - - client.handle(ev(req, null)); - - assertEquals(HttpMethod.HEAD, req.getMethod()); - } - - @Test - public void testMarker_() { - HttpRequest req = HttpRequestBuilder.get("http://test/test").build(); - - MessagePredicate predicate = FilterTestUtils.createPredicate("{\"@req.url\": {$contains: test}}"); - HttpMarkFilter f = new HttpMarkFilter(); - f.addRule(new HttpMarkFilter.MarkerRule("category1", predicate)); - ConverterHttpClient client = client(f); - - client.handle(ev(req, null)); - - List<HttpMarkFilter.Marker> markers = (List<HttpMarkFilter.Marker>) req.getTag(TAG_MARKER); - assertFalse(markers == null); - assertFalse(markers.isEmpty()); - HttpMarkFilter.Marker marker = markers.get(0); - - assertEquals("category1", marker.getCategory()); - } -}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/stress-tester/src/test/java/com/passus/st/config/TestJobConfigurator.java Wed Jun 17 11:23:46 2020 +0200 @@ -0,0 +1,102 @@ +package com.passus.st.config; + +import com.passus.commons.utils.ResourceUtils; +import com.passus.config.Configuration; +import com.passus.config.ConfigurationContext; +import com.passus.config.ConfigurationContextImpl; +import com.passus.config.YamlConfigurationReader; +import com.passus.config.validation.Errors; +import com.passus.filter.UnmutableValueExtractor; +import com.passus.filter.ValueExtractor; +import com.passus.net.PortRangeSet; +import com.passus.net.dns.session.DnsUdpSessionAnalyzer; +import com.passus.net.http.session.HttpSessionAnalyzer; +import com.passus.net.session.SessionAnalyzer; +import com.passus.net.type.NetType; +import com.passus.st.client.FlowExecutor; +import com.passus.st.emitter.Emitter; +import com.passus.st.emitter.socket.SocketEmitter; +import com.passus.st.source.EventSource; +import com.passus.st.source.NcEventSource; +import com.passus.st.source.PcapSessionEventSource; +import com.passus.st.utils.ConfigurationContextConsts; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.io.File; +import java.util.List; +import java.util.Map; + +import static org.testng.AssertJUnit.*; + +public class TestJobConfigurator { + + private final Errors errors = new Errors(); + + @BeforeClass + public void beforeClass() { + NetType.registerAll(); + } + + @AfterClass + public void afterClass() { + NetType.unregisterAll(); + } + + private ConfigurationContext processConfig(String configFileName) throws Exception { + File configFile = ResourceUtils.getFile(configFileName); + Configuration config = YamlConfigurationReader.readFromFile(configFile); + + JobConfigurator configurator = new JobConfigurator(); + ConfigurationContext context = new ConfigurationContextImpl(configFile.getParentFile()); + configurator.configure(config, errors, context); + assertFalse(errors.hasError()); + return context; + } + + @Test + public void testConfigure() throws Exception { + ConfigurationContext context = processConfig("com/passus/st/config/test_job_config.yml"); + + FlowExecutor flowExecutor = context.get(ConfigurationContextConsts.FLOW_EXECUTOR); + assertNotNull(flowExecutor); + + List<EventSource> sources = context.get(ConfigurationContextConsts.EVENT_SOURCE_SOURCES); + assertEquals(2, sources.size()); + assertTrue(sources.get(0) instanceof PcapSessionEventSource); + assertTrue(sources.get(1) instanceof NcEventSource); + + Map<String, ValueExtractor> vars = context.get(ConfigurationContextConsts.APP_VARS); + assertEquals(1, vars.size()); + UnmutableValueExtractor value = (UnmutableValueExtractor) vars.get("varName"); + assertEquals("varValue", value.getValue()); + } + + @Test + public void testConfigure_CustomPcapSourceAnalyzers() throws Exception { + ConfigurationContext context = processConfig("com/passus/st/config/test_job_custom_pcap_source_analyzers.yml"); + + List<EventSource> sources = context.get(ConfigurationContextConsts.EVENT_SOURCE_SOURCES); + assertEquals(1, sources.size()); + PcapSessionEventSource source = (PcapSessionEventSource) sources.get(0); + + List<SessionAnalyzer> analyzers = source.getAnalyzers(); + HttpSessionAnalyzer httpAnalyzer = (HttpSessionAnalyzer) analyzers.get(0); + PortRangeSet portRanges = new PortRangeSet(); + portRanges.add(801); + portRanges.add(8081); + assertEquals(portRanges, httpAnalyzer.getPortsRange()); + + assertTrue(analyzers.get(1) instanceof DnsUdpSessionAnalyzer); + } + + @Test + public void testConfigure_SocketEmitter() throws Exception { + ConfigurationContext context = processConfig("com/passus/st/config/test_job_socket_emitter.yml"); + + Emitter emitter = context.get(ConfigurationContextConsts.EMITTER_DEFAULT_EMITTER); + + assertTrue(emitter instanceof SocketEmitter); + } +}
--- a/stress-tester/src/test/java/com/passus/st/config/TestJobConfiguratorTest.java Tue Jun 16 16:53:08 2020 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,102 +0,0 @@ -package com.passus.st.config; - -import com.passus.commons.utils.ResourceUtils; -import com.passus.config.Configuration; -import com.passus.config.ConfigurationContext; -import com.passus.config.ConfigurationContextImpl; -import com.passus.config.YamlConfigurationReader; -import com.passus.config.validation.Errors; -import com.passus.filter.UnmutableValueExtractor; -import com.passus.filter.ValueExtractor; -import com.passus.net.PortRangeSet; -import com.passus.net.dns.session.DnsUdpSessionAnalyzer; -import com.passus.net.http.session.HttpSessionAnalyzer; -import com.passus.net.session.SessionAnalyzer; -import com.passus.net.type.NetType; -import com.passus.st.client.FlowExecutor; -import com.passus.st.emitter.Emitter; -import com.passus.st.emitter.socket.SocketEmitter; -import com.passus.st.source.EventSource; -import com.passus.st.source.NcEventSource; -import com.passus.st.source.PcapSessionEventSource; -import com.passus.st.utils.ConfigurationContextConsts; -import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.Test; - -import java.io.File; -import java.util.List; -import java.util.Map; - -import static org.testng.AssertJUnit.*; - -public class TestJobConfiguratorTest { - - private final Errors errors = new Errors(); - - @BeforeClass - public void beforeClass() { - NetType.registerAll(); - } - - @AfterClass - public void afterClass() { - NetType.unregisterAll(); - } - - private ConfigurationContext processConfig(String configFileName) throws Exception { - File configFile = ResourceUtils.getFile(configFileName); - Configuration config = YamlConfigurationReader.readFromFile(configFile); - - TestJobConfigurator configurator = new TestJobConfigurator(); - ConfigurationContext context = new ConfigurationContextImpl(configFile.getParentFile()); - configurator.configure(config, errors, context); - assertFalse(errors.hasError()); - return context; - } - - @Test - public void testConfigure() throws Exception { - ConfigurationContext context = processConfig("com/passus/st/config/test_job_config.yml"); - - FlowExecutor flowExecutor = context.get(ConfigurationContextConsts.FLOW_EXECUTOR); - assertNotNull(flowExecutor); - - List<EventSource> sources = context.get(ConfigurationContextConsts.EVENT_SOURCE_SOURCES); - assertEquals(2, sources.size()); - assertTrue(sources.get(0) instanceof PcapSessionEventSource); - assertTrue(sources.get(1) instanceof NcEventSource); - - Map<String, ValueExtractor> vars = context.get(ConfigurationContextConsts.APP_VARS); - assertEquals(1, vars.size()); - UnmutableValueExtractor value = (UnmutableValueExtractor) vars.get("varName"); - assertEquals("varValue", value.getValue()); - } - - @Test - public void testConfigure_CustomPcapSourceAnalyzers() throws Exception { - ConfigurationContext context = processConfig("com/passus/st/config/test_job_custom_pcap_source_analyzers.yml"); - - List<EventSource> sources = context.get(ConfigurationContextConsts.EVENT_SOURCE_SOURCES); - assertEquals(1, sources.size()); - PcapSessionEventSource source = (PcapSessionEventSource) sources.get(0); - - List<SessionAnalyzer> analyzers = source.getAnalyzers(); - HttpSessionAnalyzer httpAnalyzer = (HttpSessionAnalyzer) analyzers.get(0); - PortRangeSet portRanges = new PortRangeSet(); - portRanges.add(801); - portRanges.add(8081); - assertEquals(portRanges, httpAnalyzer.getPortsRange()); - - assertTrue(analyzers.get(1) instanceof DnsUdpSessionAnalyzer); - } - - @Test - public void testConfigure_SocketEmitter() throws Exception { - ConfigurationContext context = processConfig("com/passus/st/config/test_job_socket_emitter.yml"); - - Emitter emitter = context.get(ConfigurationContextConsts.EMITTER_DEFAULT_EMITTER); - - assertTrue(emitter instanceof SocketEmitter); - } -}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/stress-tester/src/test/java/com/passus/st/converter/ConverterJobTest.java Wed Jun 17 11:23:46 2020 +0200 @@ -0,0 +1,305 @@ +package com.passus.st.converter; + +import com.passus.net.http.*; +import com.passus.st.AppUtils; +import com.passus.st.ParametersBag; +import com.passus.st.client.FlowContext; +import com.passus.st.client.SessionPayloadEvent; +import com.passus.st.client.credentials.Credentials; +import com.passus.st.client.credentials.CredentialsProvider; +import com.passus.st.client.http.HttpScopes; +import com.passus.st.client.http.filter.*; +import com.passus.st.client.http.filter.HttpMessageModificationOperations.PostDataSetParamOperation; +import com.passus.st.client.http.filter.HttpMessageModificationOperations.SetCookieOperation; +import com.passus.st.client.http.filter.HttpMessageModificationOperations.SetHeaderOperation; +import com.passus.st.client.http.filter.HttpMessageModificationOperations.SetQueryParameterOperation; +import com.passus.st.emitter.SessionInfo; +import com.passus.st.filter.*; +import com.passus.st.source.ArrayListEventDestination; +import com.passus.st.source.NullEventDestination; +import com.passus.st.utils.TestHttpUtils; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.util.List; + +import static com.passus.commons.collection.FluentBuilder.e; +import static com.passus.commons.collection.FluentBuilder.map; +import static com.passus.st.Protocols.HTTP; +import static com.passus.st.client.http.HttpConsts.*; +import static com.passus.st.client.http.filter.HttpFlowUtils.extractHttpContext; +import static org.testng.AssertJUnit.*; + +/** + * @author mikolaj.podbielski + */ +public class ConverterJobTest { + + static SessionInfo SI = new SessionInfo("1.1.1.1", 11000, "1.1.1.2", 80, 0, HTTP); + static Credentials CREDENTIALS = new Credentials("test", "test"); + static CredentialsProvider PROVIDER = HttpAbstractLoginFilterTest.provider(CREDENTIALS); + static NullEventDestination DST = new NullEventDestination(); + + @BeforeClass + public static void beforeClass() { + AppUtils.registerAll(); + } + + @AfterClass + public static void afterClass() { + AppUtils.unregisterAll(); + } + + static SessionPayloadEvent ev(HttpRequest req, HttpResponse resp) { + return new SessionPayloadEvent(SI, req, resp, HTTP, "test"); + } + + static ConverterJob converter(FlowFilter... filters) { + ConverterJob converterJob = new ConverterJob(DST); + for (FlowFilter filter : filters) { + converterJob.filterChain.addFilter(filter); + } + converterJob.start(); + return converterJob; + } + + @Test + public void testBasicLogin_filterShouldReplaceCredentials() { + final String REQS = "GET /basic/index.html HTTP/1.1\r\n" + + "Authorization: Basic dXNlcjpwYXNzd29yZA==\r\n\r\n"; + HttpRequest req = TestHttpUtils.request(REQS); + HttpResponse resp = HttpResponseBuilder.ok().build(); + + HttpBasicAuthLoginFilter f = HttpBasicAuthLoginFilterTest.createFilter(PROVIDER); + ConverterJob converter = converter(f); + converter.handle(ev(req, resp)); + + assertEquals(req.getHeaders().get(HttpHeaders.AUTHORIZATION).toString(), "Basic dGVzdDp0ZXN0"); + + FlowContext flowContext = converter.getFlow(SI); + ParametersBag conversation = extractHttpContext(flowContext).scopes().getConversation(req); + + assertEquals(conversation.get(PARAM_USERNAME), "test"); + } + + @Test + public void testDigestLogin_filterShouldReplaceCredentials() { + final String RESP1S = "HTTP/1.1 401 Unauthorized\r\n" + + "Date: Fri, 09 Jun 2017 06:25:27 GMT\r\n" + + "WWW-Authenticate: Digest realm=\"realm\", nonce=\"old-nonce\", algorithm=MD5, qop=\"auth\"\r\n" + + "Content-Length: 12\r\n\r\nUnauthorized"; + final String REQ2S = "GET /html/digest/index2.html HTTP/1.1\r\n" + + "Host: 172.16.60.101\r\n" + + "Authorization: Digest username=\"user\", realm=\"realm\", nonce=\"old-nonce\"," + + " uri=\"/html/digest/index2.html\", algorithm=MD5, response=\"774280b22c9ed40bc500bba5c431d8c7\"," + + " qop=auth, nc=00000001, cnonce=\"cnonce\"\r\n\r\n"; + final String RESP2S = "HTTP/1.1 200 OK\n" + + "Authentication-Info: rspauth=\"?\", cnonce=\"?\", nc=00000001, qop=auth\r\n" + + "Content-Length: 33\r\n\r\n<html><body>content</body></html>"; + + HttpRequest req1 = HttpRequestBuilder.get("http://html/digest/index2.html").build(); + HttpResponse resp1 = TestHttpUtils.response(RESP1S); + HttpRequest req2 = TestHttpUtils.request(REQ2S); + HttpResponse resp2 = TestHttpUtils.response(RESP2S); + req2.setTag(TAG_SESSION_ID, "sid1"); + + HttpDigestAuthLoginFilter f = HttpDigestAuthLoginFilterTest.createFilter(CREDENTIALS); + ConverterJob converter = converter(f); + converter.handle(ev(req1, resp1)); + converter.handle(ev(req2, resp2)); + + FlowContext flowContext = converter.getFlow(SI); + HttpScopes scopes = extractHttpContext(flowContext).scopes(); + + assertEquals(scopes.getConversation(req2).get(PARAM_USERNAME), "test"); + assertEquals(scopes.getSession(req2).get(PARAM_USERNAME), "test"); + } + + @Test + public void testFormLogin() throws IOException { + final String REQS = "POST /login_check HTTP/1.1\r\n" + + "Host: 172.16.60.23\r\n" + + "Content-Type: application/x-www-form-urlencoded\r\n" + + "Content-Length: 32\r\n" + + "Cookie: PHPSESSID=4r7tg8275ih7p0uh79im6q5643\r\n\r\n" + + "_username=admin&_password=qwerty"; + final String RESPS = "HTTP/1.1 302 Found\r\n" + + "Set-Cookie: PHPSESSID=pcap_post; path=/; HttpOnly\r\n" + + "Location: http://172.16.60.23/\r\n" + + "Content-Length: 2\r\n\r\n" + + "OK"; + + HttpRequest req = TestHttpUtils.request(REQS); + HttpResponse resp = TestHttpUtils.response(RESPS); + req.setTag(TAG_SESSION_ID, "sid1"); + + HttpFormLoginFilter f = HttpFormLoginFilterTest.createFilter(PROVIDER); + ConverterJob converter = converter(f); + converter.handle(ev(req, resp)); + + FlowContext flowContext = converter.getFlow(SI); + HttpScopes scopes = extractHttpContext(flowContext).scopes(); + + HttpFormLoginFilterTest.assertContent(req, "_username=test&_password=test"); + assertEquals(scopes.getConversation(req).get(PARAM_USERNAME), "test"); + assertEquals(scopes.getSession(req).get(PARAM_USERNAME), "test"); + } + + @Test + public void testMatcher() { + HttpRequest req1 = HttpRequestBuilder.get("http://test.com/test").build(); + HttpRequest req2 = HttpRequestBuilder.get("http://test2.com/test2").build(); + HttpResponse resp1 = HttpResponseBuilder.ok().build(); + HttpResponse resp2 = HttpResponseBuilder.status(HttpStatus.NO_CONTENT).build(); + + ArrayListEventDestination dst = new ArrayListEventDestination(); + MessagePredicate predicate = FilterTestUtils.createPredicate("{'@req.url': {$contains: test2}}"); + MatchFilter f = new MatchFilter(); + f.setPredicate(predicate); + f.setAcceptOnMatch(false); + + ConverterJob conv = new ConverterJob(dst); + conv.filterChain.addFilter(f); + conv.start(); + + conv.handle(ev(req1, resp1)); // match:false filter:dunno chain:accept + conv.handle(ev(req2, resp2)); // match:true filter:deny chain:deny + + assertEquals(1, dst.getEvents().size()); + SessionPayloadEvent event = dst.findFirst(SessionPayloadEvent.class); + assertEquals(200, ((HttpResponse) event.getResponse()).getStatus().getCode()); + } + + @Test + public void testModifyMessage() throws IOException { + HttpRequest req = HttpRequestBuilder + .post("http://test.com/path/test?uParam1=1", "param1=value1¶m2=value2") + .header("Header1", "Header1Value1") + .header("Cookie", "myCookie1=myValue1;myCookie2=myValue2") + .build(); + + HttpMessageModificationFilter f = new HttpMessageModificationFilter(); + f.addOperation(new SetHeaderOperation("Header1", "Header1Value1a")); + f.addOperation(new SetCookieOperation("myCookie1", "myValue1a")); + f.addOperation(new PostDataSetParamOperation("param1", "value1a")); + f.addOperation(new SetQueryParameterOperation("uParam1", "11")); + ConverterJob conv = converter(f); + + conv.handle(ev(req, null)); + HttpHeaders headers = req.getHeaders(); + HttpParameters params = HttpMessageHelper.get().decodeFormUrlencoded(req); + + assertEquals("/path/test?uParam1=11", req.getUri().toString()); + assertEquals("Header1Value1a", headers.get("Header1").toString()); + assertEquals("myCookie1=myValue1a;myCookie2=myValue2", headers.get("Cookie").toString()); + assertEquals("param2=value2¶m1=value1a", params.toString()); + } + + @Test + public void testDateRewriter_() { + HttpRequest req = HttpRequestBuilder.get("http://example.com/") + .header(HttpHeaders.DATE, "Wed, 09-Sep-2009 09:09:00 GMT") + .build(); + + HttpDateFilter f = new HttpDateFilter(); + HttpDateFilterTest.setDate(f, 1500_000_000_000L); + ConverterJob conv = converter(f); + + conv.handle(ev(req, null)); + assertEquals(req.getHeaders().get(HttpHeaders.DATE).toString(), "Fri, 14 Jul 2017 02:40:00 GMT"); + } + + @Test(enabled = false) + public void testHostRewriter() { + HttpRequest req = HttpRequestBuilder.get("http://example.com:8080/site1/resource1") + .header(HttpHeaders.ORIGIN, "http://example.com:8080/site1") + .header(HttpHeaders.REFERER, "http://example.com:8080/site1/index") + .build(); + + HttpHostRewriterFilter f = new HttpHostRewriterFilter(); + f.setHostMap(map(e("example.com:8080", "example.org"))); + ConverterJob client = converter(f); + + //client.handle(ev(req, null)); + + HttpHeaders headers = req.getHeaders(); + assertEquals(headers.get(HttpHeaders.HOST).toString(), "example.org"); + assertEquals(headers.get(HttpHeaders.ORIGIN).toString(), "http://example.org/site1"); + assertEquals(headers.get(HttpHeaders.REFERER).toString(), "http://example.org/site1/index"); + } + + @Test + public void testHeaderCleaners() { + HttpRequest req = HttpRequestBuilder.get("http://www.example.com/abc") + .header(HttpHeaders.ACCEPT, "*/*;q=0.5") + .header(HttpHeaders.IF_MODIFIED_SINCE, "09 Sep 2009 09:09:00 GMT") + .header(HttpHeaders.X_FORWARDED_FOR, "15.16.73.33") + .build(); + HttpResponse resp = HttpResponseBuilder.ok().build(); + + HttpFilter f1 = new HttpRequestCacheHeadersCleanerFilter(); + HttpFilter f2 = new HttpRequestProxyHeadersCleanerFilter(); + ConverterJob conv = converter(f1, f2); + + conv.handle(ev(req, resp)); + HttpHeaders headers = req.getHeaders(); + + assertTrue(headers.contains(HttpHeaders.ACCEPT)); + assertFalse(headers.contains(HttpHeaders.IF_MODIFIED_SINCE)); + assertFalse(headers.contains(HttpHeaders.X_FORWARDED_FOR)); + } + + @Test + public void testZone_() { + HttpRequest req = HttpRequestBuilder.get("http://test.com/test").build(); + HttpResponse resp = HttpResponseBuilder.ok().build(); + MessagePredicate predicate = FilterTestUtils.createPredicate("{\"@req.url\": {$contains: test}}"); + + HttpZoneFilter f = new HttpZoneFilter(); + f.addRule(new HttpZoneFilter.Rule("testZone", predicate)); + ConverterJob conv = converter(f); + + conv.handle(ev(req, resp)); + + assertEquals("testZone", req.getTag(TAG_ZONE)); + assertEquals("testZone", resp.getTag(TAG_ZONE)); + } + + @Test + public void testMVEL() { + HttpRequest req = HttpRequestBuilder.get("http://example.com/index.html").build(); + + String expression = "$req.setMethod(com.passus.net.http.HttpMethod.HEAD);" + + "$req.getUri().toString().equals(\"/index.html\") ? 1 : 0"; + + MvelFilter f = new MvelFilter(); + f.setStatement(MvelFilterTest.es(expression)); + f.setDirection(FilterDirection.OUT); + ConverterJob conv = converter(f); + + conv.handle(ev(req, null)); + + assertEquals(HttpMethod.HEAD, req.getMethod()); + } + + @Test + public void testMarker_() { + HttpRequest req = HttpRequestBuilder.get("http://test/test").build(); + + MessagePredicate predicate = FilterTestUtils.createPredicate("{\"@req.url\": {$contains: test}}"); + HttpMarkFilter f = new HttpMarkFilter(); + f.addRule(new HttpMarkFilter.MarkerRule("category1", predicate)); + ConverterJob client = converter(f); + + client.handle(ev(req, null)); + + List<HttpMarkFilter.Marker> markers = (List<HttpMarkFilter.Marker>) req.getTag(TAG_MARKER); + assertFalse(markers == null); + assertFalse(markers.isEmpty()); + HttpMarkFilter.Marker marker = markers.get(0); + + assertEquals("category1", marker.getCategory()); + } +}