diff --git a/Makefile b/Makefile index 19d32c6e..b4c1914c 100644 --- a/Makefile +++ b/Makefile @@ -39,10 +39,10 @@ test_helper: docker cp . ${EVENTGEN_TEST_IMAGE}:$(shell pwd) @echo 'Verifying contents of pip.conf' - docker exec -i ${EVENTGEN_TEST_IMAGE} /bin/sh -c "cd $(shell pwd); pip install dist/splunk_eventgen*.tar.gz" + docker exec -i ${EVENTGEN_TEST_IMAGE} /bin/sh -c "cd $(shell pwd); pip3 install dist/splunk_eventgen*.tar.gz" @echo 'Installing test requirements' - docker exec -i ${EVENTGEN_TEST_IMAGE} /bin/sh -c "pip install --upgrade pip;pip install -r $(shell pwd)/requirements.txt" + docker exec -i ${EVENTGEN_TEST_IMAGE} /bin/sh -c "pip3 install --upgrade pip;pip3 install -r $(shell pwd)/requirements.txt;pip3 install git+https://github.com/esnme/ultrajson.git" @echo 'Make simulated app dir and sample for modular input test' docker exec -i ${EVENTGEN_TEST_IMAGE} /bin/sh -c "cd $(shell pwd); cd ../..; mkdir -p modinput_test_app/samples/" @@ -51,12 +51,15 @@ test_helper: @echo 'Installing docker-compose' bash install_docker_compose.sh + @echo 'Build a docker image' + docker build -t provision_splunk:latest -f tests/large/provision/Dockerfile tests/large/provision + @echo 'Start container with splunk' docker-compose -f tests/large/provision/docker-compose.yml up & sleep 120 @echo 'Provision splunk container' - docker-compose -f tests/large/provision/docker-compose.yml exec -T splunk sh -c 'cd /opt/splunk;./provision.sh;/opt/splunk/bin/splunk enable listen 9997 -auth admin:changeme;/opt/splunk/bin/splunk add index test_0;/opt/splunk/bin/splunk add index test_1;/opt/splunk/bin/splunk restart' + docker exec --user splunk provision_splunk_1 sh -c 'cd /opt/splunk;./provision.sh;./add_httpevent_collector.sh;/opt/splunk/bin/splunk enable listen 9997 -auth admin:changeme;/opt/splunk/bin/splunk add index test_0;/opt/splunk/bin/splunk add index test_1;/opt/splunk/bin/splunk restart' run_tests: @echo 'Running the super awesome tests' diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index 708bc244..9b631aa2 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -4,6 +4,8 @@ RUN apk --no-cache upgrade && \ apk add --no-cache --update \ python3 \ python3-dev \ + python2-dev \ + py2-pip \ gcc \ libc-dev \ libffi-dev \ @@ -26,7 +28,9 @@ RUN apk --no-cache upgrade && \ mkdir -p /root/.ssh && \ chmod 0700 /root/.ssh && \ passwd -u root && \ - pip3 install git+git://github.com/esnme/ultrajson.git + # install dependencies of conduct2 used by perf + pip2 install filelock twisted requests queuelib ujson psutil crochet msgpack-python unidecode attrdict service_identity && \ + pip2 install git+https://github.com/esnme/ultrajson.git COPY dockerfiles/sshd_config /etc/ssh/sshd_config COPY dockerfiles/entrypoint.sh /sbin/entrypoint.sh diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 30f02118..5c65c381 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,89 +1,58 @@ -6.5.0 -- Added metrics output mode -- Fixed regex token replacement issue -- Added test coverage information -- Increased functional test coverage -- Eventgen server complete revamp and standalone mode support -- Added contributor license -- Updated Dockerfile -- Added documentation -- Fixed bugs / stability / optimized speed - -6.4.0 -- Fixed exception log error -- Fixed CircleCI status badage error -- Fixed navigation error for app if installed with Splunk Stream -- Fixed generatorWorkers not working error -- Fixed interval error when end = 1 -- Fixed fileName in global stanza error -- Added 3rd party libs in SA-Eventgen App -- Added httpeventAllowFailureCount for httpevent -- Added 3rd party libs in license credit -- Disabled logging queue in multiprocess mode -- Changed implementation of extendIndex for better performance - -6.3.6 -- Added functional tests for jinja template and modular input feature -- Fixed default jinja template directory is not correctly resolved when sampleDir is set issue -- Fixed verbose flag not working in splunk_eventgen command line issue -- Fixed index, source, sourcetype are not correct when using splunkstream mode issue -- Fixed ssh to container issue -- Fixed perdayvolume without end setting error -- Updated documentation for better reading and remove unrelated part - -6.3.5 -- Added extendIndexes feature to support a list of indexes -- Fixed timer and token logic -- Changed end=-1 to continuously iterate without stopping -- Changed end=0 to not execute -- Added a linter for code quality -- Updated docs / docs format -- Added a suite of functional tests - -6.3.4: -- Documentation cleanup -- Jinja template bugfix in app -- Implementation of 'timeMultiple’ option -- Templates for bugs/feature requests -- Fixed Jinja test configuration stanzas -- Default behavior for 'count' edge cases - -6.3.3: -- Added performance metrics compared to Eventgen 5.x -- New config option for generation-time metrics: outputCounter -- Jinja template fixes -- Timestamp parsing fix -- Output queueing fix for outputMode splunkstream -- Count rater fixes, now supports indefinite generation - -6.3.2: -- Fixed verbosity bug -- Added documentation - -6.3.1: -- Fixed Eventgen Volume APIs -- Improved Eventgen Server Logging -- Corrected Eventgen Server and Controller conf syncing issue -- Adding verbosity options (ERROR, INFO, DEBUG) to Eventgen modinput -- Implemented future event generation support in replay mode -- Fixed Jinja template's missing default values -- Adjusted logging message levels for less verbosity -- Fixed event count off by 1 issue -- Fixed unnecessary empty data generators being created -- Updated dependency list - -6.3.0: -- Bug fixes for the customer issues -- Documentation upgrade -- Code refactoring for version unification -- Logging improvements - -6.2.1: -- Fixing SA-Eventgen Dashboard and log searching -- Improving internal logging and fixing splunkd logging issue -- Fixing timestamping in default generator -- Fixing custom plugin integration -- Fixing SA-Eventgen app settings -- Supporting Eventgen 5 backward compatibility with additional features -- Better modinput process management -- Minor Bugfixes with various customer cases +**7.1.0**: + +- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/7.1.0) + +**7.0.0**: + +- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/7.0.0) + +**6.5.2**: + +- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.5.2) + + +**6.5.1**: + +- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.5.1) + + +**6.5.0**: + +- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.5.0) + +**6.4.0**: + +- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.4.0) + +**6.3.6**: + +- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.3.6) + +**6.3.5**: + +- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.3.5) + +**6.3.4**: + +- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.3.4) + +**6.3.3**: + +- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.3.3) + +**6.3.2**: + +- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.3.2) + +**6.3.1**: + +- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.3.1) + +**6.3.0**: + +- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.3.0) + +**6.2.1**: + +- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/6.2.1) + diff --git a/docs/CONFIGURE.md b/docs/CONFIGURE.md index 449d1a8e..ea514871 100644 --- a/docs/CONFIGURE.md +++ b/docs/CONFIGURE.md @@ -323,8 +323,7 @@ Tokens in the default generator can override the sample to allow dynamic content and is a number greater than 0 and greater than or equal to . If rated, will be multiplied times hourOfDayRate and dayOfWeekRate. * For float[:], the token will be replaced with a random float between - start and end values where is a number greater than 0 - and is a number greater than 0 and greater than or equal to . + start and end values where is a number greater than or equal to . For floating point numbers, precision will be based off the precision specified in . For example, if we specify 1.0, precision will be one digit, if we specify 1.0000, precision will be four digits. If rated, will be multiplied times hourOfDayRate and dayOfWeekRate. diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock index d98655a0..df772d69 100644 --- a/docs/Gemfile.lock +++ b/docs/Gemfile.lock @@ -206,7 +206,7 @@ GEM jekyll-seo-tag (~> 2.1) minitest (5.12.2) multipart-post (2.1.1) - nokogiri (1.10.4) + nokogiri (1.10.8) mini_portile2 (~> 2.4.0) octokit (4.14.0) sawyer (~> 0.8.0, >= 0.5.3) diff --git a/docs/REFERENCE.md b/docs/REFERENCE.md index 2c9c8a76..f226cea4 100644 --- a/docs/REFERENCE.md +++ b/docs/REFERENCE.md @@ -121,17 +121,38 @@ outputWorkers = * Generally if using TCP based outputs like splunkstream, more could be required * Defaults to 1 -outputMode = modinput | s2s | file | splunkstream | stdout | devnull | spool | httpevent | syslogout | tcpout | udpout | metric_httpevent +outputMode = scsout | modinput | s2s | file | splunkstream | stdout | devnull | spool | httpevent | syslogout | tcpout | udpout | metric_httpevent * Specifies how to output log data. Modinput is default. + * If setting scsout, should set scsEndPoint and scsAccessToken. scsClientId, scsClientSecret, and scsRetryNum are optional. * If setting spool, should set spoolDir * If setting file, should set fileName * If setting splunkstream, should set splunkHost, splunkPort, splunkMethod, splunkUser and splunkPassword if not Splunk embedded * If setting s2s, should set splunkHost and splunkPort - * If setting syslogout, should set syslogDestinationHost and syslogDestinationPort + * If setting syslogout, should set syslogDestinationHost and syslogDestinationPort. A UDP port listening on Splunk needs to be configured. https://docs.splunk.com/Documentation/Splunk/latest/Data/HowSplunkEnterprisehandlessyslogdata * If setting httpevent, should set httpeventServers * If setting metric_httpevent, should set httpeventServers and make sure your index is a splunk metric index +scsEndPoint = + * Should be a full url to the scs endpoint + +scsAccessToken = + * Should be a scs access token. Do not include "Bearer". + +scsClientId = + * Optional + * SCS client id that is used to renew the access token if it expires during the data generation + * If not supplied, will not renew the access token and data transmission might fail + +scsClientSecret = + * Optional + * SCS client secret that is used to renew the access token if it expires during the data generation + * If not supplied, will not renew the access token and data transmission might fail + +scsRetryNum = + * Optional and defaults to 0 + * Retry a failing data transmission batch + syslogDestinationHost = * Defaults to 127.0.0.1 @@ -543,8 +564,7 @@ token..replacement = | | ["list","of","strptime"] | guid and is a number greater than 0 and greater than or equal to . If rated, will be multiplied times hourOfDayRate and dayOfWeekRate. * For float[:], the token will be replaced with a random float between - start and end values where is a number greater than 0 - and is a number greater than 0 and greater than or equal to . + start and end values where is a number greater than or equal to . For floating point numbers, precision will be based off the precision specified in . For example, if we specify 1.0, precision will be one digit, if we specify 1.0000, precision will be four digits. If rated, diff --git a/release_tool/README.md b/release_tool/README.md new file mode 100644 index 00000000..9cbecfe7 --- /dev/null +++ b/release_tool/README.md @@ -0,0 +1,15 @@ +# Release tool + +Use script to bump the release verison and create the release PR to merge to develop branch. + +**Note: this script only works with python3.** + +- If you have generated your github access token, you can use the following command to bump versions and send PR automatically. + ```bash + python prepare_release_branch.py -v -n -a + ``` + +- If the access token is not given, this script only is only used to bump the release version and push the commit to remote repo. You need to go to github web page to create your PR manually. + ``` + python prepare_release_branch.py -v -n + ``` diff --git a/release_tool/prepare_release_branch.py b/release_tool/prepare_release_branch.py new file mode 100644 index 00000000..d227bc61 --- /dev/null +++ b/release_tool/prepare_release_branch.py @@ -0,0 +1,176 @@ +import argparse +import os +import sys +import logging +import json +import re +import subprocess +import requests + +logging.getLogger().setLevel(logging.INFO) +root_repo_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def parse_args(): + def validate_version_str(version): + v_str = str(version).strip() + if not v_str: + raise argparse.ArgumentTypeError('verison str can not be emtpy.') + err_message = 'version string should be of format "major.minor.hotfix"' + numbers = v_str.split('.') + if len(numbers) != 3: + raise argparse.ArgumentTypeError(err_message) + for n in numbers: + valid = False + try: + v = int(n) + valid = (v >= 0) + except: + valid = False + if not valid: + raise argparse.ArgumentTypeError(err_message) + return v_str + + def validate_token(token): + t = token.strip() + if not t: + raise argparse.ArgumentTypeError('token can not be empty') + return t + + parser = argparse.ArgumentParser( + 'prepare_release_branch.py', description= + 'eventgen release branch tool.\ncreate the release branch, set the right version and push the pull request.') + parser.add_argument('-v', '--verbose', default=False, action='store_true', help='enable the verbose logging') + parser.add_argument('-n', '--version_str', type=validate_version_str, required=True) + parser.add_argument('-a', '--token', help='your github access token.', default=None, type=validate_token) + return parser.parse_args(sys.argv[1:]) + + +def setup_logging(verbose=None): + l = logging.DEBUG if verbose is True else logging.INFO + logging.getLogger().setLevel(l) + + +def setup_env(): + ''' + by default, we use this hard code current working dir. + because curent working dir has impact about the child sh process. + we need to setup it before launching any process. + if there is concrete requirement about setting the current + working dir, we can change it to cmd arguemnt. + ''' + logging.debug(f'try to change current working directory to {root_repo_dir}') + os.chdir(root_repo_dir) + + +def run_sh_cmd(args, exit_on_error=None): + should_exit_on_error = True if exit_on_error is None else exit_on_error + child = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = child.communicate() + outs = out.decode('utf-8') + errs = err.decode('utf-8') + if child.returncode == 0: + logging.debug(f'execute sh command {args} success.') + logging.debug(f'children output:\n{outs}') + return True + logging.error(f'execute sh cmd {args} fail.\nchildren output:\n{outs}\n{errs}') + if should_exit_on_error: + assert False, 'sh command fails.' + return False + + +def get_release_branch_name(version_str): + v = version_str.replace('.', '_') + return f'release/{v}' + + +def replace_version(ver): + ver_json_file = os.path.join(root_repo_dir, 'splunk_eventgen', 'version.json') + with open(ver_json_file, 'w') as fp: + json.dump({'version': ver}, fp) + app_conf = os.path.join(root_repo_dir, 'splunk_eventgen', 'splunk_app', 'default', 'app.conf') + app_conf_content = [] + with open(app_conf, 'r') as fp: + app_conf_content = fp.readlines() + app_pattern = re.compile(r'version\s*=') + with open(app_conf, 'w') as fp: + for line in app_conf_content: + lstr = line.strip() + if app_pattern.search(lstr): + fp.write(f'version = {ver}\n') + else: + fp.write(f'{lstr}\n') + logging.info(f'verison is replaced with {ver}.') + + +def update_changelog(ver): + changelog_file = os.path.join(root_repo_dir, 'docs', 'CHANGELOG.md') + content = None + with open(changelog_file, 'r') as fp: + content = fp.readlines() + new_content = f'**{ver}**:\n\n' + f'- Check the release note and download the package/source from [Here](https://github.com/splunk/eventgen/releases/tag/{ver})\n\n' + with open(changelog_file, 'w') as fp: + fp.write(new_content) + for l in content: + fp.write(l) + logging.info('CHANGELOG.md is updated.') + + +def commit_updated_files(ver): + ver_json_file = os.path.join('splunk_eventgen', 'version.json') + app_conf = os.path.join('splunk_eventgen', 'splunk_app', 'default', 'app.conf') + changelog = os.path.join('docs', 'CHANGELOG.md') + run_sh_cmd(['git', 'add', ver_json_file]) + run_sh_cmd(['git', 'add', app_conf]) + run_sh_cmd(['git', 'add', changelog]) + run_sh_cmd(['git', 'commit', '-m', f'update eventgen version to {ver}'], False) + logging.info('committed version files.') + + +def create_pr(ver, token, target_branch): + release_branch = get_release_branch_name(ver) + response = requests.post( + 'https://api.github.com/repos/splunk/eventgen/pulls', json={ + 'title': f'Release eventgen {ver}. Merge to {target_branch} branch.', 'head': release_branch, 'base': + target_branch, 'body': 'As the title'}, headers={ + 'Accept': 'application/vnd.github.full+json', 'Content-Type': 'application/json', 'Authorization': + f'token {token}'}) + response.raise_for_status() + data = response.json() + pr_url = data['url'] + logging.info(f'Pull request is created:\n\t{pr_url}') + + +if __name__ == '__main__': + arg_values = parse_args() + if arg_values is None: + sys.exit(1) + setup_logging(arg_values.verbose) + setup_env() + + logging.info('checkout to the develop branch and pull the latest change...') + run_sh_cmd(['git', 'checkout', 'develop']) + run_sh_cmd(['git', 'pull']) + + logging.info('check out the release branch') + release_branch = get_release_branch_name(arg_values.version_str) + branch_exist = run_sh_cmd(['git', 'show-ref', '--verify', f'refs/heads/{release_branch}'], False) + if not branch_exist: + run_sh_cmd(['git', 'checkout', '-b', release_branch]) + else: + run_sh_cmd(['git', 'checkout', release_branch]) + + replace_version(arg_values.version_str) + update_changelog(arg_values.version_str) + + commit_updated_files(arg_values.version_str) + + run_sh_cmd(['git', 'push', 'origin', release_branch]) + logging.info(f'release branch {release_branch} is pushed to remote repo.') + + if arg_values.token: + create_pr(arg_values.version_str, arg_values.token, 'develop') + create_pr(arg_values.version_str, arg_values.token, 'master') + else: + pr_url = 'https://github.com/splunk/eventgen/compare' + logging.info('create pull reqeust manually by visiting this url:\n{pr_url}') diff --git a/requirements.txt b/requirements.txt index fd0bfe6a..46c88479 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ pytest==4.6.4 pytest-xdist mock pytest-cov +coverage==4.5.4 docker==3.7.3 pyOpenSSL lxml==4.3.4 @@ -12,7 +13,7 @@ requests[security] ujson>=1.35 pyyaml httplib2 -jinja2 +jinja2==2.10.3 urllib3==1.24.2 pyOpenSSL flake8>=3.7.7 diff --git a/splunk_eventgen/README/eventgen.conf.tutorial1 b/splunk_eventgen/README/eventgen.conf.tutorial1 index 282d978e..4ea72bcf 100644 --- a/splunk_eventgen/README/eventgen.conf.tutorial1 +++ b/splunk_eventgen/README/eventgen.conf.tutorial1 @@ -5,21 +5,21 @@ timeMultiple = 2 outputMode = stdout token.0.token = \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3,6} -token.0.replacementType = timestamp +token.0.replacementType = replaytimestamp token.0.replacement = %Y-%m-%d %H:%M:%S,%f token.1.token = \d{2}-\d{2}-\d{4} \d{2}:\d{2}:\d{2}.\d{3,6} -token.1.replacementType = timestamp +token.1.replacementType = replaytimestamp token.1.replacement = %m-%d-%Y %H:%M:%S.%f token.2.token = \d{2}/\w{3}/\d{4}:\d{2}:\d{2}:\d{2}.\d{3,6} -token.2.replacementType = timestamp +token.2.replacementType = replaytimestamp token.2.replacement = %d/%b/%Y:%H:%M:%S.%f token.3.token = \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} -token.3.replacementType = timestamp +token.3.replacementType = replaytimestamp token.3.replacement = %Y-%m-%d %H:%M:%S token.4.token = \d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2} -token.4.replacementType = timestamp +token.4.replacementType = replaytimestamp token.4.replacement = %Y-%m-%dT%H:%M:%S diff --git a/splunk_eventgen/__init__.py b/splunk_eventgen/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/splunk_eventgen/__main__.py b/splunk_eventgen/__main__.py index ce66a6d0..9a43bc41 100644 --- a/splunk_eventgen/__main__.py +++ b/splunk_eventgen/__main__.py @@ -76,6 +76,7 @@ def parse_args(): service_subparser.add_argument("--redis-host", type=str, default='127.0.0.1', help="Redis Host") service_subparser.add_argument("--redis-port", type=str, default='6379', help="Redis Port") service_subparser.add_argument("--web-server-port", type=str, default='9500', help="Port you want to run a web server on") + service_subparser.add_argument("--multithread", action="store_true", help="Use multi-thread instead of multi-process") # Help subparser # NOTE: Keep this at the end so we can use the subparser_dict.keys() to display valid commands help_subparser = subparsers.add_parser('help', help="Display usage on a subcommand") @@ -202,6 +203,8 @@ def gather_env_vars(args): env_vars["REDIS_HOST"] = os.environ.get("REDIS_HOST", args.redis_host) env_vars["REDIS_PORT"] = os.environ.get("REDIS_PORT", args.redis_port) env_vars["WEB_SERVER_PORT"] = os.environ.get("WEB_SERVER_PORT", args.web_server_port) + if "multithread" in args: + env_vars["multithread"] = args.multithread return env_vars diff --git a/splunk_eventgen/eventgen_api_server/eventgen_core_object.py b/splunk_eventgen/eventgen_api_server/eventgen_core_object.py index 87c89a26..9b0eb6a1 100644 --- a/splunk_eventgen/eventgen_api_server/eventgen_core_object.py +++ b/splunk_eventgen/eventgen_api_server/eventgen_core_object.py @@ -9,9 +9,9 @@ class EventgenCoreObject: - def __init__(self): + def __init__(self, **kargs): self.logger = logging.getLogger('eventgen_server') - self.eventgen_core_object = eventgen_core.EventGenerator(self._create_args()) + self.eventgen_core_object = eventgen_core.EventGenerator(self._create_args(**kargs)) self.configured = False self.configfile = None self.check_and_configure_eventgen() @@ -24,26 +24,26 @@ def check_and_configure_eventgen(self): self.logger.info("Configured Eventgen from {}".format(CUSTOM_CONFIG_PATH)) def refresh_eventgen_core_object(self): - self.eventgen_core_object.kill_processes() - self.eventgen_core_object = eventgen_core.EventGenerator(self._create_args()) + self.eventgen_core_object.stop(force_stop=True) self.configured = False self.configfile = None self.check_and_configure_eventgen() self.logger.info("Refreshed the eventgen core object") - def _create_args(self): + def _create_args(self, **kargs): args = argparse.Namespace() args.daemon = False args.version = False args.backfill = None args.count = None + args.end = None args.devnull = False args.disableOutputQueue = False args.generators = None args.interval = None args.keepoutput = False args.modinput = False - args.multiprocess = True + args.multiprocess = False if kargs.get("multithread") else True args.outputters = None args.profiler = False args.sample = None diff --git a/splunk_eventgen/eventgen_api_server/eventgen_server.py b/splunk_eventgen/eventgen_api_server/eventgen_server.py index f95493c8..7146694a 100644 --- a/splunk_eventgen/eventgen_api_server/eventgen_server.py +++ b/splunk_eventgen/eventgen_api_server/eventgen_server.py @@ -8,10 +8,10 @@ class EventgenServer: - def __init__(self, *args, **kwargs): - self.eventgen = eventgen_core_object.EventgenCoreObject() - self.mode = kwargs.get('mode', 'standalone') + def __init__(self, *args, **kwargs): self.env_vars = kwargs.get('env_vars') + self.eventgen = eventgen_core_object.EventgenCoreObject(mutithread=self.env_vars.get("multithread", False)) + self.mode = kwargs.get('mode', 'standalone') self.host = socket.gethostname() self.role = 'server' diff --git a/splunk_eventgen/eventgen_api_server/eventgen_server_api.py b/splunk_eventgen/eventgen_api_server/eventgen_server_api.py index d9133d45..eab43b1d 100644 --- a/splunk_eventgen/eventgen_api_server/eventgen_server_api.py +++ b/splunk_eventgen/eventgen_api_server/eventgen_server_api.py @@ -34,7 +34,8 @@ def __init__(self, eventgen, redis_connector, host, mode='standalone'): self.host = host self.interval = 0.01 - if mode != 'standalone': + self.mode = mode + if self.mode != 'standalone': self.redis_connector = redis_connector self._channel_listener() self.logger.info("Initialized the channel listener. Cluster mode ready.") @@ -163,12 +164,7 @@ def http_post_start(): @bp.route('/stop', methods=['POST']) def http_post_stop(): try: - force_stop = False - try: - force_stop = True - except: - force_stop = False - response = self.stop(force_stop = force_stop) + response = self.stop(force_stop=True) self.eventgen.refresh_eventgen_core_object() return Response(json.dumps(response), mimetype='application/json', status=200) except Exception as e: @@ -433,13 +429,16 @@ def reset(self): def healthcheck(self): response = {} - try: - self.redis_connector.pubsub.check_health() - response['message'] = "Connections are healthy" - except Exception as e: - self.logger.error("Connection to Redis failed: {}, re-registering".format(str(e))) - self.redis_connector.register_myself(hostname=self.host, role="server") - response['message'] = "Connections unhealthy - re-established connections" + if self.mode != 'standalone': + try: + self.redis_connector.pubsub.check_health() + response['message'] = "Connections are healthy" + except Exception as e: + self.logger.error("Connection to Redis failed: {}, re-registering".format(str(e))) + self.redis_connector.register_myself(hostname=self.host, role="server") + response['message'] = "Connections unhealthy - re-established connections" + else: + response['message'] = "Standalone {} is healthy".format(self.host) return response def set_bundle(self, url): @@ -546,7 +545,7 @@ def setup_http(self, data): del kv_pair['httpeventServers'] conf_dict['global']['threading'] = 'process' conf_dict['global']['httpeventMaxPayloadSize'] = '256000' - conf_dict['global']['outputMode'] = 'httpevent' + conf_dict['global']['outputMode'] = data.get("outputMode") if data.get("outputMode") else 'httpevent' conf_dict['global']['httpeventServers'] = {"servers": data.get("servers")} self.set_conf(conf_dict) else: @@ -616,6 +615,6 @@ def create_new_hec_key(hostname): del kv_pair['httpeventServers'] conf_dict['global']['threading'] = 'process' conf_dict['global']['httpeventMaxPayloadSize'] = '256000' - conf_dict['global']['outputMode'] = 'httpevent' + conf_dict['global']['outputMode'] = data.get("outputMode") if data.get("outputMode") else 'httpevent' conf_dict['global']['httpeventServers'] = {"servers": self.discovered_servers} self.set_conf(conf_dict) diff --git a/splunk_eventgen/eventgen_core.py b/splunk_eventgen/eventgen_core.py index 32d3763f..48721951 100644 --- a/splunk_eventgen/eventgen_core.py +++ b/splunk_eventgen/eventgen_core.py @@ -8,7 +8,7 @@ import time from queue import Empty, Queue import signal -from threading import Thread +from threading import Thread, Event import multiprocessing from splunk_eventgen.lib.eventgenconfig import Config @@ -32,13 +32,14 @@ def __init__(self, args=None): localized .conf entries. :param args: __main__ parse_args() object. ''' - self.stopping = False + self.stop_request = Event() self.force_stop = False self.started = False self.completed = False self.config = None self.args = args - + self.workerPool = [] + self.manager = None self._setup_loggers(args=args) # attach to the logging queue self.logger.info("Logging Setup Complete.") @@ -94,9 +95,6 @@ def _load_config(self, configfile, **kwargs): else: generator_worker_count = self.config.generatorWorkers - # TODO: Probably should destroy pools better so processes are cleaned. - if self.args.multiprocess: - self.kill_processes() self._setup_pools(generator_worker_count) def _reload_plugins(self): @@ -192,7 +190,7 @@ def _create_generator_pool(self, workercount=20): ''' if self.args.multiprocess: self.manager = multiprocessing.Manager() - if self.config.disableLoggingQueue: + if self.config and self.config.disableLoggingQueue: self.loggingQueue = None else: # TODO crash caused by logging Thread https://github.com/splunk/eventgen/issues/217 @@ -236,6 +234,7 @@ def _create_generator_workers(self, workercount=20): )) self.workerPool.append(process) process.start() + self.logger.info("create process: {}".format(process.pid)) else: pass @@ -252,7 +251,7 @@ def _setup_loggers(self, args=None): self.logger.setLevel(logging.ERROR) def _worker_do_work(self, work_queue, logging_queue): - while not self.stopping: + while not self.stop_request.isSet(): try: item = work_queue.get(timeout=10) startTime = time.time() @@ -263,12 +262,15 @@ def _worker_do_work(self, work_queue, logging_queue): work_queue.task_done() except Empty: pass + except EOFError as ef: + self.logger.exception(str(ef)) + continue except Exception as e: self.logger.exception(str(e)) raise e def _generator_do_work(self, work_queue, logging_queue, output_counter=None): - while not self.stopping: + while not self.stop_request.isSet(): try: item = work_queue.get(timeout=10) startTime = time.time() @@ -279,6 +281,9 @@ def _generator_do_work(self, work_queue, logging_queue, output_counter=None): work_queue.task_done() except Empty: pass + except EOFError as ef: + self.logger.exception(str(ef)) + continue except Exception as e: if self.force_stop: break @@ -308,8 +313,9 @@ def _proc_worker_do_work(work_queue, logging_queue, config, disable_logging): item._out.updateConfig(item.config) item.run() work_queue.task_done() + item.logger.info("Current Worker Stopping: {0}".format(stopping)) + item.logger = None stopping = genconfig['stopping'] - item.logger.debug("Current Worker Stopping: {0}".format(stopping)) except Empty: stopping = genconfig['stopping'] except Exception as e: @@ -320,7 +326,7 @@ def _proc_worker_do_work(work_queue, logging_queue, config, disable_logging): sys.exit(0) def logger_thread(self, loggingQueue): - while not self.stopping: + while not self.stop_request.isSet(): try: record = loggingQueue.get(timeout=10) logger.handle(record) @@ -414,7 +420,7 @@ def _initializePlugins(self, dirname, plugins, plugintype, name=None): return ret def start(self, join_after_start=True): - self.stopping = False + self.stop_request.clear() self.started = True self.config.stopping = False self.completed = False @@ -454,23 +460,19 @@ def join_process(self): raise e def stop(self, force_stop=False): - # empty the sample queue: - self.config.stopping = True - self.stopping = True + if hasattr(self.config, "stopping"): + self.config.stopping = True self.force_stop = force_stop + # set the thread event to stop threads + self.stop_request.set() - self.logger.info("All timers exited, joining generation queue until it's empty.") - if force_stop: - self.logger.info("Forcibly stopping Eventgen: Deleting workerQueue.") - del self.workerQueue - self._create_generator_pool() - self.workerQueue.join() # if we're in multiprocess, make sure we don't add more generators after the timers stopped. if self.args.multiprocess: if force_stop: self.kill_processes() else: - self.genconfig["stopping"] = True + if hasattr(self, "genconfig"): + self.genconfig["stopping"] = True for worker in self.workerPool: count = 0 # We wait for a minute until terminating the worker @@ -484,12 +486,9 @@ def stop(self, force_stop=False): time.sleep(2) count += 1 - self.logger.info("All generators working/exited, joining output queue until it's empty.") - if not self.args.multiprocess and not force_stop: - self.outputQueue.join() - self.logger.info("All items fully processed. Cleaning up internal processes.") self.started = False - self.stopping = False + # clear the thread event + self.stop_request.clear() def reload_conf(self, configfile): ''' @@ -534,14 +533,14 @@ def check_done(self): return self.sampleQueue.empty() and self.sampleQueue.unfinished_tasks <= 0 and self.workerQueue.empty() def kill_processes(self): - try: - if self.args.multiprocess: - for worker in self.workerPool: - try: - os.kill(int(worker.pid), signal.SIGKILL) - except: - continue - del self.outputQueue - self.manager.shutdown() - except: - pass + self.logger.info("Kill worker processes") + for worker in self.workerPool: + try: + self.logger.info("Kill worker process: {}".format(worker.pid)) + os.kill(int(worker.pid), signal.SIGKILL) + except Exception as e: + self.logger.ERROR(str(e)) + continue + self.workerPool = [] + if self.manager: + self.manager.shutdown() diff --git a/splunk_eventgen/lib/eventgenconfig.py b/splunk_eventgen/lib/eventgenconfig.py index 3acca7c5..d9078f9a 100644 --- a/splunk_eventgen/lib/eventgenconfig.py +++ b/splunk_eventgen/lib/eventgenconfig.py @@ -1,3 +1,4 @@ +import copy import datetime import json import logging.handlers @@ -520,15 +521,22 @@ def parse(self): if os.path.exists(s.sampleDir): sampleFiles = os.listdir(s.sampleDir) for sample in sampleFiles: - results = re.match(s.name, sample) + sample_name = s.name + # If we expect a .csv, append it to the file name - regex matching must include the extension + if s.sampletype == "csv" and not s.name.endswith(".csv"): + sample_name = s.name + "\.csv" + results = re.match(sample_name, sample) if results: - logger.debug("Matched file {0} with sample name {1}".format(results.group(0), s.name)) - samplePath = os.path.join(s.sampleDir, sample) - if os.path.isfile(samplePath): - logger.debug( - "Found sample file '%s' for app '%s' using config '%s' with priority '%s'" % - (sample, s.app, s.name, s._priority) + "; adding to list") - foundFiles.append(samplePath) + # Make sure the stanza name/regex matches the entire file name + match_start, match_end = results.regs[0] + if match_end - match_start == len(sample): + logger.debug("Matched file {0} with sample name {1}".format(results.group(0), s.name)) + samplePath = os.path.join(s.sampleDir, sample) + if os.path.isfile(samplePath): + logger.debug( + "Found sample file '%s' for app '%s' using config '%s' with priority '%s'" % + (sample, s.app, s.name, s._priority) + "; adding to list") + foundFiles.append(samplePath) # If we didn't find any files, log about it if len(foundFiles) == 0: @@ -539,8 +547,8 @@ def parse(self): tempsamples2.append(s) for f in foundFiles: - if s.name in f: - news = s + if re.search(s.name, f): + news = copy.copy(s) news.filePath = f # 12/3/13 CS TODO These are hard coded but should be handled via the modular config system # Maybe a generic callback for all plugins which will modify sample based on the filename diff --git a/splunk_eventgen/lib/eventgenoutput.py b/splunk_eventgen/lib/eventgenoutput.py index 1290e38b..4399989c 100644 --- a/splunk_eventgen/lib/eventgenoutput.py +++ b/splunk_eventgen/lib/eventgenoutput.py @@ -71,21 +71,6 @@ def flush(self, endOfInterval=False): Flushes output buffer, unless endOfInterval called, and then only flush if we've been called more than maxIntervalsBeforeFlush tunable. """ - # TODO: Fix interval flushing somehow with a queue, not sure I even want to support this feature anymore. - '''if endOfInterval: - logger.debugv("Sample calling flush, checking increment against maxIntervalsBeforeFlush") - c.intervalsSinceFlush[self._sample.name].increment() - if c.intervalsSinceFlush[self._sample.name].value() >= self._sample.maxIntervalsBeforeFlush: - logger.debugv("Exceeded maxIntervalsBeforeFlush, flushing") - flushing = True - c.intervalsSinceFlush[self._sample.name].clear() - else: - logger.debugv("Not enough events to flush, passing flush routine.") - else: - logger.debugv("maxQueueLength exceeded, flushing") - flushing = True''' - - # TODO: This is set this way just for the time being while I decide if we want this feature. flushing = True if flushing: q = self._queue @@ -113,3 +98,4 @@ def flush(self, endOfInterval=False): self._sample.name, 'events': len(tmp), 'bytes': sum(tmp)}) tmp = None outputer.run() + q = None diff --git a/splunk_eventgen/lib/eventgentimer.py b/splunk_eventgen/lib/eventgentimer.py index 90af68b4..d374ee81 100644 --- a/splunk_eventgen/lib/eventgentimer.py +++ b/splunk_eventgen/lib/eventgentimer.py @@ -108,6 +108,7 @@ def real_run(self): # referenced in the config object, while, self.stopping will only stop this one. if self.config.stopping or self.stopping: end = True + continue count = self.rater.rate() # First run of the generator, see if we have any backfill work to do. if self.countdown <= 0: diff --git a/splunk_eventgen/lib/eventgentoken.py b/splunk_eventgen/lib/eventgentoken.py index e3351e05..433602e7 100644 --- a/splunk_eventgen/lib/eventgentoken.py +++ b/splunk_eventgen/lib/eventgentoken.py @@ -349,7 +349,7 @@ def _getReplacement(self, old=None, earliestTime=None, latestTime=None, s=None, except: logger.error("Could not parse json for '%s' in sample '%s'" % (listMatch.group(1), s.name)) return old - return random.SystemRandom().choice(value) + return random.choice(value) else: logger.error("Unknown replacement value '%s' for replacementType '%s'; will not replace" % diff --git a/splunk_eventgen/lib/generatorplugin.py b/splunk_eventgen/lib/generatorplugin.py index 1c190079..a736ee2c 100644 --- a/splunk_eventgen/lib/generatorplugin.py +++ b/splunk_eventgen/lib/generatorplugin.py @@ -37,6 +37,8 @@ def build_events(self, eventsDict, startTime, earliest, latest, ignore_tokens=Fa """Ready events for output by replacing tokens and updating the output queue""" # Replace tokens first so that perDayVolume evaluates the correct event length send_objects = self.replace_tokens(eventsDict, earliest, latest, ignore_tokens=ignore_tokens) + # after replace_tokens() is called, we don't need eventsDict + del eventsDict try: self._out.bulksend(send_objects) self._sample.timestamp = None diff --git a/splunk_eventgen/lib/logging_config/__init__.py b/splunk_eventgen/lib/logging_config/__init__.py index 5a0e1c29..d602589b 100644 --- a/splunk_eventgen/lib/logging_config/__init__.py +++ b/splunk_eventgen/lib/logging_config/__init__.py @@ -2,7 +2,7 @@ import logging.config LOG_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'logs') -DEFAULT_LOGGING_LEVEL = "DEBUG" +DEFAULT_LOGGING_LEVEL = "ERROR" LOGGING_CONFIG = { 'version': 1, @@ -76,7 +76,7 @@ 'loggers': { 'eventgen': { - 'handlers': ['console', 'eventgen_main'], + 'handlers': ['eventgen_main'], 'level': DEFAULT_LOGGING_LEVEL, 'propagate': False }, diff --git a/splunk_eventgen/lib/outputplugin.py b/splunk_eventgen/lib/outputplugin.py index 663042dc..fbb72c8a 100644 --- a/splunk_eventgen/lib/outputplugin.py +++ b/splunk_eventgen/lib/outputplugin.py @@ -42,6 +42,5 @@ def run(self): def _output_end(self): pass - def load(): return OutputPlugin diff --git a/splunk_eventgen/lib/plugins/generator/perdayvolumegenerator.py b/splunk_eventgen/lib/plugins/generator/perdayvolumegenerator.py index 8a7fdd5e..10bc8159 100644 --- a/splunk_eventgen/lib/plugins/generator/perdayvolumegenerator.py +++ b/splunk_eventgen/lib/plugins/generator/perdayvolumegenerator.py @@ -80,7 +80,7 @@ def gen(self, count, earliest, latest, samplename=None): (self._sample.name, self._sample.app, len(eventsDict))) # build the events and replace tokens - GeneratorPlugin.build_events(self, eventsDict, startTime, earliest, latest) + self.build_events(eventsDict, startTime, earliest, latest) def load(): diff --git a/splunk_eventgen/lib/plugins/generator/replay.py b/splunk_eventgen/lib/plugins/generator/replay.py index 7cb7092d..dc80b6d3 100644 --- a/splunk_eventgen/lib/plugins/generator/replay.py +++ b/splunk_eventgen/lib/plugins/generator/replay.py @@ -112,7 +112,7 @@ def gen(self, count, earliest, latest, samplename=None): continue # Refer to the last event to calculate the new backfill time - time_difference = datetime.timedelta(seconds=(current_event_timestamp - previous_event_timestamp) .total_seconds() * self._sample.timeMultiple) + time_difference = datetime.timedelta(seconds=(current_event_timestamp - previous_event_timestamp).total_seconds() * self._sample.timeMultiple) if self.backfill_time + time_difference >= self.current_time: sleep_time = time_difference - (self.current_time - self.backfill_time) diff --git a/splunk_eventgen/lib/plugins/output/httpevent.py b/splunk_eventgen/lib/plugins/output/httpevent.py index 704e26d2..e33b0eb0 100644 --- a/splunk_eventgen/lib/plugins/output/httpevent.py +++ b/splunk_eventgen/lib/plugins/output/httpevent.py @@ -77,6 +77,7 @@ def flush(self, q): payload.append(payloadFragment) logger.debug("Finished processing events, sending all to splunk") self._sendHTTPEvents(payload) + payload = [] if self.config.httpeventWaitResponse: for session in self.active_sessions: response = session.result() diff --git a/splunk_eventgen/lib/plugins/output/httpevent_core.py b/splunk_eventgen/lib/plugins/output/httpevent_core.py index 56eff9c8..bd47abc7 100644 --- a/splunk_eventgen/lib/plugins/output/httpevent_core.py +++ b/splunk_eventgen/lib/plugins/output/httpevent_core.py @@ -176,7 +176,7 @@ def _sendHTTPEvents(self, payload): try: self._transmitEvents(stringpayload) totalbytessent += len(stringpayload) - currentreadsize = 0 + currentreadsize = targetlinesize stringpayload = targetline except Exception as e: logger.exception(str(e)) @@ -207,7 +207,6 @@ def _transmitEvents(self, payloadstring): headers['content-type'] = 'application/json' try: payloadsize = len(payloadstring) - # response = requests.post(url, data=payloadstring, headers=headers, verify=False) self.active_sessions.append( self.session.post(url=url, data=payloadstring, headers=headers, verify=False)) except Exception as e: diff --git a/splunk_eventgen/lib/plugins/output/scsout.py b/splunk_eventgen/lib/plugins/output/scsout.py new file mode 100644 index 00000000..6a66c367 --- /dev/null +++ b/splunk_eventgen/lib/plugins/output/scsout.py @@ -0,0 +1,166 @@ +from splunk_eventgen.lib.outputplugin import OutputPlugin +from splunk_eventgen.lib.logging_config import logger + +import logging +import requests +import time +import sys +import os + +import requests +from requests import Session +from requests_futures.sessions import FuturesSession +from concurrent.futures import ThreadPoolExecutor + +try: + import ujson as json +except: + import json + +class NoSCSEndPoint(Exception): + def __init__(self, *args, **kwargs): + Exception.__init__(self, *args, **kwargs) + +class NoSCSAccessToken(Exception): + def __init__(self, *args, **kwargs): + Exception.__init__(self, *args, **kwargs) + +class SCSOutputPlugin(OutputPlugin): + useOutputQueue = False + name = 'scsout' + MAXQUEUELENGTH = 1000 + + def __init__(self, sample, output_counter=None): + OutputPlugin.__init__(self, sample, output_counter) + + self.scsHttpPayloadMax = 150000 # Documentation recommends 20KB to 200KB. Going with 150KB. + self.scsEndPoint = getattr(self._sample, "scsEndPoint", None) + self.scsAccessToken = getattr(self._sample, "scsAccessToken", None) + self.scsClientId = getattr(self._sample, 'scsClientId', '') + self.scsClientSecret = getattr(self._sample, 'scsClientSecret', '') + self.scsRetryNum = int(getattr(self._sample, 'scsRetryNum', 0)) # By default, retry num is 0 + + self._setup_REST_workers() + + def _setup_REST_workers(self, session=None, workers=10): + # disable any "requests" warnings + requests.packages.urllib3.disable_warnings() + # Bind passed in samples to the outputter. + if not session: + session = Session() + self.session = FuturesSession(session=session, executor=ThreadPoolExecutor(max_workers=workers)) + self.active_sessions = [] + + def flush(self, events): + if not self.scsEndPoint: + if getattr(self.config, 'scsEndPoint', None): + self.scsEndPoint = self.config.scsEndPoint + else: + raise NoSCSEndPoint("Please specify your REST endpoint for the SCS tenant") + + if not self.scsAccessToken: + if getattr(self.config, 'scsAccessToken', None): + self.scsAccessToken = self.config.scsAccessToken + else: + raise NoSCSAccessToken("Please specify your REST endpoint access token for the SCS tenant") + + if self.scsClientId and self.scsClientSecret: + logger.info("Both scsClientId and scsClientSecret are supplied. We will renew the expired token using these credentials.") + self.scsRenewToken = True + else: + if getattr(self.config, 'scsClientId', None) and getattr(self.config, 'scsClientSecret', None): + self.scsClientId = self.config.scsClientId + self.scsClientSecret = self.config.scsClientSecret + logger.info("Both scsClientId and scsClientSecret are supplied. We will renew the expired token using these credentials.") + self.scsRenewToken = True + else: + self.scsRenewToken = False + + self.header = { + "Authorization": f"Bearer {self.scsAccessToken}", + "Content-Type": "application/json" + } + + self.accessTokenExpired = False + self.tokenRenewEndPoint = "https://auth.scp.splunk.com/token" + self.tokenRenewBody = { + "client_id": self.scsClientId, + "client_secret": self.scsClientSecret, + "grant_type": "client_credentials" + } + + for i in range(self.scsRetryNum + 1): + logger.debug(f"Sending data to the scs endpoint. Num:{i}") + self._sendHTTPEvents(events) + + if not self.checkResults(): + if self.accessTokenExpired and self.scsRenewToken: + self.renewAccessToken() + self.active_sessions = [] + else: + break + + def checkResults(self): + for session in self.active_sessions: + response = session.result() + if response.status_code == 401 and "Invalid or Expired Bearer Token" in response.text: + logger.error("scsAccessToken is invalid or expired") + self.accessTokenExpired = True + return False + elif response.status_code != 200: + logger.error(f"Data transmisison failed with {response.status_code} and {response.text}") + return False + logger.debug(f"Data transmission successful") + return True + + def renewAccessToken(self): + response = requests.post(self.tokenRenewEndPoint, data=self.tokenRenewBody, timeout=5) + if response.status_code == 200: + logger.info("Renewal of the access token succesful") + self.scsAccessToken = response.json()["access_token"] + setattr(self._sample, "scsAccessToken", self.scsAccessToken) + self.accessTokenExpired = False + else: + logger.error("Renewal of the access token failed") + + def _sendHTTPEvents(self, events): + currentPayloadSize = 0 + currentPayload = [] + try: + for event in events: + # Reformat the event to fit the scs request spec + # TODO: Move this logic to generator + try: + event["body"] = event.pop("_raw") + event["timestamp"] = int(event.pop("_time") * 1000) + event.pop("index") + if "attributes" not in event: + event["attributes"] = {} + event["attributes"]["hostRegex"] = event.pop("hostRegex") + except: + pass + + targetline = json.dumps(event) + targetlinesize = len(targetline) + + # Continue building a current payload if the payload is less than the max size + if (currentPayloadSize + targetlinesize) < self.scsHttpPayloadMax: + currentPayload.append(event) + currentPayloadSize += targetlinesize + else: + self.active_sessions.append(self.session.post(url=self.scsEndPoint, data=json.dumps(currentPayload), headers=self.header, verify=False)) + currentPayloadSize = targetlinesize + currentPayload = [event] + + # Final flush of the leftover events + if currentPayloadSize > 0: + self.active_sessions.append(self.session.post(url=self.scsEndPoint, data=json.dumps(currentPayload), headers=self.header, verify=False)) + + except Exception as e: + logger.exception(str(e)) + raise e + + +def load(): + """Returns an instance of the plugin""" + return SCSOutputPlugin \ No newline at end of file diff --git a/splunk_eventgen/lib/plugins/output/stdout.py b/splunk_eventgen/lib/plugins/output/stdout.py index 54734b5c..8471bc27 100644 --- a/splunk_eventgen/lib/plugins/output/stdout.py +++ b/splunk_eventgen/lib/plugins/output/stdout.py @@ -11,7 +11,7 @@ def __init__(self, sample, output_counter=None): def flush(self, q): for x in q: - print(x['_raw'].rstrip()) + print(x.get('_raw', '').rstrip()) def load(): diff --git a/splunk_eventgen/lib/plugins/output/syslogout.py b/splunk_eventgen/lib/plugins/output/syslogout.py index 31267aa9..e63b269a 100644 --- a/splunk_eventgen/lib/plugins/output/syslogout.py +++ b/splunk_eventgen/lib/plugins/output/syslogout.py @@ -17,6 +17,7 @@ def filter(self, record): record.host = self.host return True + class SyslogOutOutputPlugin(OutputPlugin): useOutputQueue = True name = 'syslogout' diff --git a/splunk_eventgen/lib/plugins/output/tcpout.py b/splunk_eventgen/lib/plugins/output/tcpout.py index 5843072d..488099b0 100644 --- a/splunk_eventgen/lib/plugins/output/tcpout.py +++ b/splunk_eventgen/lib/plugins/output/tcpout.py @@ -22,7 +22,8 @@ def flush(self, q): self.s.connect((self._tcpDestinationHost, int(self._tcpDestinationPort))) logger.info("Socket connected to {0}:{1}".format(self._tcpDestinationHost, self._tcpDestinationPort)) for x in q: - self.s.send(x['_raw'].rstrip() + '\n') + msg = x['_raw'].rstrip() + '\n' + self.s.send(str.encode(msg)) self.s.close() diff --git a/splunk_eventgen/lib/plugins/output/udpout.py b/splunk_eventgen/lib/plugins/output/udpout.py index 02699acd..8e3632e5 100644 --- a/splunk_eventgen/lib/plugins/output/udpout.py +++ b/splunk_eventgen/lib/plugins/output/udpout.py @@ -21,7 +21,7 @@ def __init__(self, sample, output_counter=None): def flush(self, q): for x in q: msg = x['_raw'].rstrip() + '\n' - self.s.sendto(msg, (self._udpDestinationHost, int(self._udpDestinationPort))) + self.s.sendto(str.encode(msg), (self._udpDestinationHost, int(self._udpDestinationPort))) logger.info("Flushing in udpout.") diff --git a/splunk_eventgen/lib/requirements.txt b/splunk_eventgen/lib/requirements.txt index 4bfb4b0b..51638740 100644 --- a/splunk_eventgen/lib/requirements.txt +++ b/splunk_eventgen/lib/requirements.txt @@ -1,2 +1,4 @@ ujson==1.35 -jinja2==2.10.1 +jinja2==2.10.3 +requests-futures==1.0.0 +urllib3==1.24.2 diff --git a/splunk_eventgen/splunk_app/README/eventgen.conf.spec b/splunk_eventgen/splunk_app/README/eventgen.conf.spec index c713f2f5..ac23eef6 100644 --- a/splunk_eventgen/splunk_app/README/eventgen.conf.spec +++ b/splunk_eventgen/splunk_app/README/eventgen.conf.spec @@ -467,8 +467,7 @@ token..replacement = | | ["list","of","strptime"] | guid and is a number greater than 0 and greater than or equal to . If rated, will be multiplied times hourOfDayRate and dayOfWeekRate. * For float[:], the token will be replaced with a random float between - start and end values where is a number greater than 0 - and is a number greater than 0 and greater than or equal to . + start and end values where is a number greater than or equal to . For floating point numbers, precision will be based off the precision specified in . For example, if we specify 1.0, precision will be one digit, if we specify 1.0000, precision will be four digits. If rated, will be multiplied times hourOfDayRate and dayOfWeekRate. diff --git a/splunk_eventgen/splunk_app/default/app.conf b/splunk_eventgen/splunk_app/default/app.conf index 880a9a25..3bb576d3 100644 --- a/splunk_eventgen/splunk_app/default/app.conf +++ b/splunk_eventgen/splunk_app/default/app.conf @@ -14,7 +14,7 @@ build = 1 [launcher] author = Splunk Inc. -version = 6.3.2 +version = 7.1.0 description = SA-Eventgen app for dynamic data generation [package] diff --git a/splunk_eventgen/version.json b/splunk_eventgen/version.json index 342b591f..dde1c1e3 100644 --- a/splunk_eventgen/version.json +++ b/splunk_eventgen/version.json @@ -1 +1 @@ -{"version": "7.0.0"} +{"version": "7.1.0"} \ No newline at end of file diff --git a/tests/large/conf/eventgen_sample_regex_csv.conf b/tests/large/conf/eventgen_sample_regex_csv.conf new file mode 100755 index 00000000..2544c2bb --- /dev/null +++ b/tests/large/conf/eventgen_sample_regex_csv.conf @@ -0,0 +1,15 @@ +[timeorder.*] +sampleDir = ../sample +mode = sample +sampletype = csv +outputMode = stdout +end = 1 + +token.0.token = \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} +token.0.replacementType = timestamp +token.0.replacement = %Y-%m-%d %H:%M:%S + +token.1.token = @@integer +token.1.replacementType = random +token.1.replacement = integer[0:10] + diff --git a/tests/large/conf/eventgen_sample_regex_integer.conf b/tests/large/conf/eventgen_sample_regex_integer.conf new file mode 100644 index 00000000..9af999b6 --- /dev/null +++ b/tests/large/conf/eventgen_sample_regex_integer.conf @@ -0,0 +1,15 @@ +[sample\d] +sampleDir = ../sample +mode = sample +earliest = -15s +sampletype = raw +outputMode = stdout +end = 1 + +token.0.token = \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} +token.0.replacementType = timestamp +token.0.replacement = %Y-%m-%d %H:%M:%S + +token.1.token = @@integer +token.1.replacementType = random +token.1.replacement = integer[0:10] diff --git a/tests/large/conf/eventgen_sample_regex_wildcard.conf b/tests/large/conf/eventgen_sample_regex_wildcard.conf new file mode 100644 index 00000000..b57676aa --- /dev/null +++ b/tests/large/conf/eventgen_sample_regex_wildcard.conf @@ -0,0 +1,15 @@ +[sample.*] +sampleDir = ../sample +mode = sample +earliest = -15s +sampletype = raw +outputMode = stdout +end = 1 + +token.0.token = \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} +token.0.replacementType = timestamp +token.0.replacement = %Y-%m-%d %H:%M:%S + +token.1.token = @@integer +token.1.replacementType = random +token.1.replacement = integer[0:10] diff --git a/tests/large/provision/Dockerfile b/tests/large/provision/Dockerfile index 3aa91ead..12ea2b09 100644 --- a/tests/large/provision/Dockerfile +++ b/tests/large/provision/Dockerfile @@ -1,11 +1,8 @@ -FROM splunk/splunk:7.0.3-monitor +FROM splunk/splunk:7.3-debian -# https://superuser.com/questions/1423486/issue-with-fetching-http-deb-debian-org-debian-dists-jessie-updates-inrelease -RUN printf "deb http://archive.debian.org/debian/ jessie main\ndeb-src http://archive.debian.org/debian/ jessie main\ndeb http://security.debian.org jessie/updates main\ndeb-src http://security.debian.org jessie/updates main" > /etc/apt/sources.list - -RUN apt-get update +RUN sudo apt-get update RUN echo "installing docker dependencies and development tools" && \ - apt-get --assume-yes install curl vim + sudo apt-get --assume-yes install curl vim -COPY provision.sh /opt/splunk/ +COPY ["provision.sh", "add_httpevent_collector.sh", "/opt/splunk/"] diff --git a/tests/large/provision/add_httpevent_collector.sh b/tests/large/provision/add_httpevent_collector.sh new file mode 100755 index 00000000..1f574e5f --- /dev/null +++ b/tests/large/provision/add_httpevent_collector.sh @@ -0,0 +1,5 @@ +HTTP_INPUTS_PATH=/opt/splunk/etc/apps/search/local/inputs.conf +echo "[http://test]" >> $HTTP_INPUTS_PATH +echo "disabled = 0" >> $HTTP_INPUTS_PATH +echo "token = 00000000-0000-0000-0000-000000000000" >> $HTTP_INPUTS_PATH +echo "indexes = main,test_0,test_1" >> $HTTP_INPUTS_PATH diff --git a/tests/large/provision/docker-compose.yml b/tests/large/provision/docker-compose.yml index f8165a8e..1762a7f6 100644 --- a/tests/large/provision/docker-compose.yml +++ b/tests/large/provision/docker-compose.yml @@ -2,7 +2,7 @@ version: "3.3" services: splunk: hostname: eventgensplunk - build: . + image: provision_splunk:latest ports: - 8000:8000 - 8089:8089 @@ -12,8 +12,10 @@ services: SPLUNK_START_ARGS: --answer-yes --no-prompt --accept-license # add `SHELL` env variable to make the `dircolors` happy SHELL: /bin/bash + SPLUNK_PASSWORD: changeme volumes: # the `docker` command in guest can talk to host docker daemon - "/var/run/docker.sock:/var/run/docker.sock" # to make terminal colorful tty: true + \ No newline at end of file diff --git a/tests/large/sample/sample1 b/tests/large/sample/sample1 new file mode 100755 index 00000000..b906dc98 --- /dev/null +++ b/tests/large/sample/sample1 @@ -0,0 +1,12 @@ +2014-01-04 20:00:00 WINDBAG Event 1 of 12 randint @@integer +2014-01-04 20:00:01 WINDBAG Event 2 of 12 randint @@integer +2014-01-04 20:00:02 WINDBAG Event 3 of 12 randint @@integer +2014-01-04 20:00:03 WINDBAG Event 4 of 12 randint @@integer +2014-01-04 20:00:03 WINDBAG Event 5 of 12 randint @@integer +2014-01-04 20:00:04 WINDBAG Event 6 of 12 randint @@integer +2014-01-04 20:00:05 WINDBAG Event 7 of 12 randint @@integer +2014-01-04 20:00:06 WINDBAG Event 8 of 12 randint @@integer +2014-01-04 20:00:08 WINDBAG Event 9 of 12 randint @@integer +2014-01-04 20:00:20 WINDBAG Event 10 of 12 randint @@integer +2014-01-04 20:00:21 WINDBAG Event 11 of 12 randint @@integer +2014-01-04 20:00:21 WINDBAG Event 12 of 12 randint @@integer diff --git a/tests/large/sample/sample2 b/tests/large/sample/sample2 new file mode 100755 index 00000000..b906dc98 --- /dev/null +++ b/tests/large/sample/sample2 @@ -0,0 +1,12 @@ +2014-01-04 20:00:00 WINDBAG Event 1 of 12 randint @@integer +2014-01-04 20:00:01 WINDBAG Event 2 of 12 randint @@integer +2014-01-04 20:00:02 WINDBAG Event 3 of 12 randint @@integer +2014-01-04 20:00:03 WINDBAG Event 4 of 12 randint @@integer +2014-01-04 20:00:03 WINDBAG Event 5 of 12 randint @@integer +2014-01-04 20:00:04 WINDBAG Event 6 of 12 randint @@integer +2014-01-04 20:00:05 WINDBAG Event 7 of 12 randint @@integer +2014-01-04 20:00:06 WINDBAG Event 8 of 12 randint @@integer +2014-01-04 20:00:08 WINDBAG Event 9 of 12 randint @@integer +2014-01-04 20:00:20 WINDBAG Event 10 of 12 randint @@integer +2014-01-04 20:00:21 WINDBAG Event 11 of 12 randint @@integer +2014-01-04 20:00:21 WINDBAG Event 12 of 12 randint @@integer diff --git a/tests/large/sample/timeorderXcsv b/tests/large/sample/timeorderXcsv new file mode 100644 index 00000000..72e2fc31 --- /dev/null +++ b/tests/large/sample/timeorderXcsv @@ -0,0 +1,11 @@ +_time,_raw,index,host,source,sourcetype +2015-08-18T16:28:54.695-0700,"127.0.0.1 - admin [18/Aug/2015:16:28:54.695 -0700] ""GET /en-US/api/shelper?snippet=true&snippetEmbedJS=false&namespace=search&search=search+index%3D_internal+%7C+fields+_time%2C+_raw%2C+index%2C+host%2C+source%2C+sourcetype+&useTypeahead=true&useAssistant=true&showCommandHelp=true&showCommandHistory=true&showFieldInfo=false&_=1439940537886 HTTP/1.1"" 200 994 ""https://host5.foobar.com:8000/en-US/app/search/search?q=search%20index%3D_internal%20%7C%20fields%20_time%2C%20_raw%2C%20index%2C%20host%2C%20source%2C%20sourcetype&sid=1439940529.1846224&earliest=&latest="" ""Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.132 Safari/537.36"" - 55d3bfb6b17f7ff8270d50 33ms",_internal,host5.foobar.com,/usr/local/bamboo/itsi-demo/local/splunk/var/log/splunk/web_access.log,splunk_web_access +2015-08-18T16:28:54.569-0700,"2015-08-18 16:28:54,569 INFO streams_utils:24 - utils::readAsJson:: /usr/local/bamboo/itsi-demo/local/splunk/etc/apps/splunk_app_stream/local/apps",_internal,host5.foobar.com,/usr/local/bamboo/itsi-demo/local/splunk/var/log/splunk/splunk_app_stream.log,splunk_app_stream.log +2015-08-18T16:28:54.568-0700,"2015-08-18 16:28:54,568 INFO streams_utils:74 - create dir /usr/local/bamboo/itsi-demo/local/splunk/etc/apps/splunk_app_stream/local/",_internal,host5.foobar.com,/usr/local/bamboo/itsi-demo/local/splunk/var/log/splunk/splunk_app_stream.log,splunk_app_stream.log +2015-08-18T16:28:54.564-0700,"127.0.0.1 - - [18/Aug/2015:16:28:54.564 -0700] ""GET /en-us/custom/splunk_app_stream/ping/ HTTP/1.1"" 200 311 """" """" - 55d3bfb6907f7ff805f710 5ms",_internal,host5.foobar.com,/usr/local/bamboo/itsi-demo/local/splunk/var/log/splunk/web_access.log,splunk_web_access +2015-08-18T16:28:52.798-0700,"10.160.255.115 - admin [18/Aug/2015:16:28:52.798 -0700] ""GET /en-US/splunkd/__raw/servicesNS/nobody/search/search/jobs/1439940529.1846224/summary?output_mode=json&min_freq=0&_=1439940537880 HTTP/1.1"" 200 503 ""https://host5.foobar.com:8000/en-US/app/search/search?q=search%20index%3D_internal%20%7C%20fields%20_time%2C%20_raw%2C%20index%2C%20host%2C%20source%2C%20sourcetype&sid=1439940529.1846224&earliest=&latest="" ""Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.132 Safari/537.36"" - 9f802569d5c3d77d468e897d34f8969f 6ms",_internal,host5.foobar.com,/usr/local/bamboo/itsi-demo/local/splunk/var/log/splunk/splunkd_ui_access.log,splunkd_ui_access +2015-08-18T16:28:52.798-0700,"10.160.255.115 - admin [18/Aug/2015:16:28:52.798 -0700] ""GET /en-US/splunkd/__raw/services/search/jobs/1439940529.1846224/timeline?offset=0&count=1000&_=1439940537881 HTTP/1.1"" 200 349 ""https://host5.foobar.com:8000/en-US/app/search/search?q=search%20index%3D_internal%20%7C%20fields%20_time%2C%20_raw%2C%20index%2C%20host%2C%20source%2C%20sourcetype&sid=1439940529.1846224&earliest=&latest="" ""Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.132 Safari/537.36"" - 9f802569d5c3d77d468e897d34f8969f 4ms",_internal,host5.foobar.com,/usr/local/bamboo/itsi-demo/local/splunk/var/log/splunk/splunkd_ui_access.log,splunkd_ui_access +2015-08-18T16:28:52.754-0700,"10.160.255.115 - admin [18/Aug/2015:16:28:52.754 -0700] ""GET /en-US/splunkd/__raw/servicesNS/nobody/search/search/jobs/1439940529.1846224?output_mode=json&_=1439940537879 HTTP/1.1"" 200 1543 ""https://host5.foobar.com:8000/en-US/app/search/search?q=search%20index%3D_internal%20%7C%20fields%20_time%2C%20_raw%2C%20index%2C%20host%2C%20source%2C%20sourcetype&sid=1439940529.1846224&earliest=&latest="" ""Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.132 Safari/537.36"" - 9f802569d5c3d77d468e897d34f8969f 4ms",_internal,host5.foobar.com,/usr/local/bamboo/itsi-demo/local/splunk/var/log/splunk/splunkd_ui_access.log,splunkd_ui_access +2015-08-18T16:28:52.270-0700,"2015-08-18 16:28:52,270 ERROR pid=16324 tid=MainThread file=__init__.py:execute:957 | Execution failed: [HTTP 401] Client is not authenticated +2015-08-18T16:28:52.268-0700,"127.0.0.1 - - [18/Aug/2015:16:28:52.268 -0700] ""GET /services/shcluster/config/config HTTP/1.0"" 401 148 - - - 0ms",_internal,host5.foobar.com,/usr/local/bamboo/itsi-demo/local/splunk/var/log/splunk/splunkd_access.log,splunkd_access +2015-08-18T16:28:52.247-0700,"2015-08-18 16:28:52,247 INFO pid=16324 tid=MainThread file=__init__.py:execute:906 | Execute called",_internal,host5.foobar.com,/usr/local/bamboo/itsi-demo/local/splunk/var/log/splunk/python_modular_input.log,python_modular_input diff --git a/tests/large/sample/timeorder_regex.csv b/tests/large/sample/timeorder_regex.csv new file mode 100644 index 00000000..72e2fc31 --- /dev/null +++ b/tests/large/sample/timeorder_regex.csv @@ -0,0 +1,11 @@ +_time,_raw,index,host,source,sourcetype +2015-08-18T16:28:54.695-0700,"127.0.0.1 - admin [18/Aug/2015:16:28:54.695 -0700] ""GET /en-US/api/shelper?snippet=true&snippetEmbedJS=false&namespace=search&search=search+index%3D_internal+%7C+fields+_time%2C+_raw%2C+index%2C+host%2C+source%2C+sourcetype+&useTypeahead=true&useAssistant=true&showCommandHelp=true&showCommandHistory=true&showFieldInfo=false&_=1439940537886 HTTP/1.1"" 200 994 ""https://host5.foobar.com:8000/en-US/app/search/search?q=search%20index%3D_internal%20%7C%20fields%20_time%2C%20_raw%2C%20index%2C%20host%2C%20source%2C%20sourcetype&sid=1439940529.1846224&earliest=&latest="" ""Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.132 Safari/537.36"" - 55d3bfb6b17f7ff8270d50 33ms",_internal,host5.foobar.com,/usr/local/bamboo/itsi-demo/local/splunk/var/log/splunk/web_access.log,splunk_web_access +2015-08-18T16:28:54.569-0700,"2015-08-18 16:28:54,569 INFO streams_utils:24 - utils::readAsJson:: /usr/local/bamboo/itsi-demo/local/splunk/etc/apps/splunk_app_stream/local/apps",_internal,host5.foobar.com,/usr/local/bamboo/itsi-demo/local/splunk/var/log/splunk/splunk_app_stream.log,splunk_app_stream.log +2015-08-18T16:28:54.568-0700,"2015-08-18 16:28:54,568 INFO streams_utils:74 - create dir /usr/local/bamboo/itsi-demo/local/splunk/etc/apps/splunk_app_stream/local/",_internal,host5.foobar.com,/usr/local/bamboo/itsi-demo/local/splunk/var/log/splunk/splunk_app_stream.log,splunk_app_stream.log +2015-08-18T16:28:54.564-0700,"127.0.0.1 - - [18/Aug/2015:16:28:54.564 -0700] ""GET /en-us/custom/splunk_app_stream/ping/ HTTP/1.1"" 200 311 """" """" - 55d3bfb6907f7ff805f710 5ms",_internal,host5.foobar.com,/usr/local/bamboo/itsi-demo/local/splunk/var/log/splunk/web_access.log,splunk_web_access +2015-08-18T16:28:52.798-0700,"10.160.255.115 - admin [18/Aug/2015:16:28:52.798 -0700] ""GET /en-US/splunkd/__raw/servicesNS/nobody/search/search/jobs/1439940529.1846224/summary?output_mode=json&min_freq=0&_=1439940537880 HTTP/1.1"" 200 503 ""https://host5.foobar.com:8000/en-US/app/search/search?q=search%20index%3D_internal%20%7C%20fields%20_time%2C%20_raw%2C%20index%2C%20host%2C%20source%2C%20sourcetype&sid=1439940529.1846224&earliest=&latest="" ""Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.132 Safari/537.36"" - 9f802569d5c3d77d468e897d34f8969f 6ms",_internal,host5.foobar.com,/usr/local/bamboo/itsi-demo/local/splunk/var/log/splunk/splunkd_ui_access.log,splunkd_ui_access +2015-08-18T16:28:52.798-0700,"10.160.255.115 - admin [18/Aug/2015:16:28:52.798 -0700] ""GET /en-US/splunkd/__raw/services/search/jobs/1439940529.1846224/timeline?offset=0&count=1000&_=1439940537881 HTTP/1.1"" 200 349 ""https://host5.foobar.com:8000/en-US/app/search/search?q=search%20index%3D_internal%20%7C%20fields%20_time%2C%20_raw%2C%20index%2C%20host%2C%20source%2C%20sourcetype&sid=1439940529.1846224&earliest=&latest="" ""Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.132 Safari/537.36"" - 9f802569d5c3d77d468e897d34f8969f 4ms",_internal,host5.foobar.com,/usr/local/bamboo/itsi-demo/local/splunk/var/log/splunk/splunkd_ui_access.log,splunkd_ui_access +2015-08-18T16:28:52.754-0700,"10.160.255.115 - admin [18/Aug/2015:16:28:52.754 -0700] ""GET /en-US/splunkd/__raw/servicesNS/nobody/search/search/jobs/1439940529.1846224?output_mode=json&_=1439940537879 HTTP/1.1"" 200 1543 ""https://host5.foobar.com:8000/en-US/app/search/search?q=search%20index%3D_internal%20%7C%20fields%20_time%2C%20_raw%2C%20index%2C%20host%2C%20source%2C%20sourcetype&sid=1439940529.1846224&earliest=&latest="" ""Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.132 Safari/537.36"" - 9f802569d5c3d77d468e897d34f8969f 4ms",_internal,host5.foobar.com,/usr/local/bamboo/itsi-demo/local/splunk/var/log/splunk/splunkd_ui_access.log,splunkd_ui_access +2015-08-18T16:28:52.270-0700,"2015-08-18 16:28:52,270 ERROR pid=16324 tid=MainThread file=__init__.py:execute:957 | Execution failed: [HTTP 401] Client is not authenticated +2015-08-18T16:28:52.268-0700,"127.0.0.1 - - [18/Aug/2015:16:28:52.268 -0700] ""GET /services/shcluster/config/config HTTP/1.0"" 401 148 - - - 0ms",_internal,host5.foobar.com,/usr/local/bamboo/itsi-demo/local/splunk/var/log/splunk/splunkd_access.log,splunkd_access +2015-08-18T16:28:52.247-0700,"2015-08-18 16:28:52,247 INFO pid=16324 tid=MainThread file=__init__.py:execute:906 | Execute called",_internal,host5.foobar.com,/usr/local/bamboo/itsi-demo/local/splunk/var/log/splunk/python_modular_input.log,python_modular_input diff --git a/tests/large/sample/tutorial1 b/tests/large/sample/tutorial1.csv similarity index 100% rename from tests/large/sample/tutorial1 rename to tests/large/sample/tutorial1.csv diff --git a/tests/large/test_mode_replay.py b/tests/large/test_mode_replay.py index b0576852..441b1de2 100644 --- a/tests/large/test_mode_replay.py +++ b/tests/large/test_mode_replay.py @@ -52,7 +52,7 @@ def test_mode_replay_backfill_greater_interval(eventgen_test_helper): def test_mode_replay_tutorial1(eventgen_test_helper): - """Test the replay mode with csv for sample file sample.tutorial1. https://github.com/splunk/eventgen/issues/244""" + """Test the replay mode with csv for sample file sample.tutorial1.csv""" events = eventgen_test_helper('eventgen_tutorial1.conf').get_events() assert len(events) == 2019 diff --git a/tests/large/test_mode_sample.py b/tests/large/test_mode_sample.py index daea33ff..f86db600 100644 --- a/tests/large/test_mode_sample.py +++ b/tests/large/test_mode_sample.py @@ -105,3 +105,21 @@ def test_mode_sample_generator_workers(eventgen_test_helper): """Test sample mode with generatorWorkers = 5, end = 5 and count = 10""" events = eventgen_test_helper("eventgen_sample_generatorWorkers.conf").get_events() assert len(events) == 50 + + +def test_mode_sample_regex_integer(eventgen_test_helper): + """Test sample mode with a regex pattern in the stanza name ('sample\d')""" + events = eventgen_test_helper("eventgen_sample_regex_integer.conf").get_events() + assert len(events) == 24 + + +def test_mode_sample_regex_wildcard(eventgen_test_helper): + """tTest sample mode with a regex wildcard pattern in the stanza name ('sample*')""" + events = eventgen_test_helper("eventgen_sample_regex_wildcard.conf").get_events() + assert len(events) == 36 + + +def test_mode_sample_regex_csv(eventgen_test_helper): + """tTest sample mode with a regex wildcard pattern in the stanza name ('sample*')""" + events = eventgen_test_helper("eventgen_sample_regex_csv.conf").get_events() + assert len(events) == 20 diff --git a/tests/medium/plugins/test_scs_output.py b/tests/medium/plugins/test_scs_output.py new file mode 100644 index 00000000..46d899cc --- /dev/null +++ b/tests/medium/plugins/test_scs_output.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +import os +import sys +import requests + +from mock import MagicMock, patch + +from splunk_eventgen.__main__ import parse_args +from splunk_eventgen.eventgen_core import EventGenerator +from splunk_eventgen.lib.plugins.output.scsout import SCSOutputPlugin + +FILE_DIR = os.path.dirname(os.path.abspath(__file__)) + + +class TestSCSOutputPlugin(object): + def test_output_data_to_scs(self): + configfile = "tests/sample_eventgen_conf/medium_test/eventgen.conf.scsoutput" + testargs = ["eventgen", "generate", configfile] + with patch.object(sys, 'argv', testargs): + pargs = parse_args() + assert pargs.subcommand == 'generate' + assert pargs.configfile == configfile + eventgen = EventGenerator(args=pargs) + with patch('requests_futures.sessions.FuturesSession.post') as mock_requests: + sample = MagicMock() + scsoutput = SCSOutputPlugin(sample) + + eventgen.start() + scsoutput.session.post.assert_called() + assert scsoutput.session.post.call_count == 1 diff --git a/tests/sample_eventgen_conf/medium_test/eventgen.conf.scsoutput b/tests/sample_eventgen_conf/medium_test/eventgen.conf.scsoutput new file mode 100644 index 00000000..79dda19e --- /dev/null +++ b/tests/sample_eventgen_conf/medium_test/eventgen.conf.scsoutput @@ -0,0 +1,14 @@ +[windbag] +generator = windbag +earliest = -3s +latest = now +interval = 3 +count = 5 +end = 1 +outputMode = scsout +host = eventgen_scs_plugin +source = scs_plugin_test +sourcetype = scs_plugin_test_type + +scsEndPoint = http://127.0.0.1 +scsAccessToken = testToken diff --git a/tests/sample_eventgen_conf/scsout/eventgen.conf b/tests/sample_eventgen_conf/scsout/eventgen.conf new file mode 100644 index 00000000..d16ec6d5 --- /dev/null +++ b/tests/sample_eventgen_conf/scsout/eventgen.conf @@ -0,0 +1,16 @@ +[splunk_cloud_platform_events.txt] +sampleDir = . +interval = 1 +mode = replay +end = 1 +outputMode = scsout +host = eventgen_scs_plugin +source = scs_plugin_test +sourcetype = scs_plugin_test_type + +scsEndPoint = +scsAccessToken = + +token.0.token = \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} +token.0.replacementType = replaytimestamp +token.0.replacement = %Y-%m-%d %H:%M:%S \ No newline at end of file diff --git a/tests/sample_eventgen_conf/scsout/splunk_cloud_platform_events.txt b/tests/sample_eventgen_conf/scsout/splunk_cloud_platform_events.txt new file mode 100644 index 00000000..5a3a8f05 --- /dev/null +++ b/tests/sample_eventgen_conf/scsout/splunk_cloud_platform_events.txt @@ -0,0 +1,5 @@ +2014-01-04 20:00:00 Event1 happened +2014-01-04 20:00:01 Event2 happened +2014-01-04 20:00:03 Event3 happened +2014-01-04 20:00:05 Event4 happened user bought @@item +2014-01-04 20:00:10 Event5 happened @@item \ No newline at end of file diff --git a/tests/sample_eventgen_conf/windbag/eventgen.conf.windbag b/tests/sample_eventgen_conf/windbag/eventgen.conf.windbag index b6e4292e..1a7a8c98 100644 --- a/tests/sample_eventgen_conf/windbag/eventgen.conf.windbag +++ b/tests/sample_eventgen_conf/windbag/eventgen.conf.windbag @@ -1,8 +1,10 @@ [windbag] generator = windbag -earliest = -3s -latest = now interval = 3 -count = 10 +count = 5 end = 3 -outputMode = stdout \ No newline at end of file +outputMode = stdout + +token.0.token = \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} +token.0.replacementType = replaytimestamp +token.0.replacement = %Y-%m-%d %H:%M:%S \ No newline at end of file diff --git a/tests/sample_jinja_addon.zip b/tests/sample_jinja_addon.zip new file mode 100644 index 00000000..2b958e6a Binary files /dev/null and b/tests/sample_jinja_addon.zip differ