Skip to content

Commit

Permalink
Merge branch 'v0.2.x' into merge-v0.2.x-test
Browse files Browse the repository at this point in the history
  • Loading branch information
@JunAishima
JunAishima committed Oct 3, 2024
2 parents 06311ea + 512abaf commit 3c5b566
Show file tree
Hide file tree
Showing 17 changed files with 356 additions and 413 deletions.
73 changes: 73 additions & 0 deletions .github/workflows/testing.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
name: tests

on:
push:
pull_request:
# schedule:
# - cron: '00 4 * * *' # daily at 4AM

jobs:
build:
name: Test conftrack with Python ${{ matrix.python-version }} (${{ matrix.dependencies }})
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"]
dependencies: ["pip", "conda"]
fail-fast: false

defaults:
run:
shell: bash -l {0}

steps:
- name: Set env vars
run: |
export REPOSITORY_NAME=${GITHUB_REPOSITORY#*/} # just the repo, as opposed to org/repo
echo "REPOSITORY_NAME=${REPOSITORY_NAME}" >> $GITHUB_ENV
- name: Checkout the code
uses: actions/checkout@v2

- name: Set up Python ${{ matrix.python-version }} with conda
uses: conda-incubator/setup-miniconda@v2
with:
activate-environment: ${{ env.REPOSITORY_NAME }}-py${{ matrix.python-version }}
auto-update-conda: true
miniconda-version: "latest"
python-version: ${{ matrix.python-version }}

- name: Install auxiliary dependencies
run: |
set -vxeo pipefail
python3 -m pip install --upgrade pip wheel
- name: Install dependencies with ${{ matrix.dependencies }}
if: matrix.dependencies == 'pip'
run: |
set -vxeo pipefail
python3 -m pip install -r requirements.txt
python3 -m pip install -r requirements-dev.txt
- name: Install dependencies with ${{ matrix.dependencies }}
if: matrix.dependencies == 'conda'
run: |
set -vxeo pipefail
conda install -y -c conda-forge doct jsonschema mongomock mongoquery pymongo pytest pyyaml requests six tornado ujson
- name: Install the package
run: |
set -vxeo pipefail
python3 -m pip install . -vv --no-deps
- name: Check installed dependencies
run: |
set -vxeo pipefail
conda env list
pip list
conda list
- name: Test with pytest
run: |
set -vxeuo pipefail
pytest -s -vv
5 changes: 2 additions & 3 deletions amostra/connection.yml
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
# amostra configuration
# EDIT
mongo_host: localhost
mongo_port: 27017
mongo_uri: mongodb://localhost:27017
timezone: US/Eastern
protocol: http
service_port: 7770
service_port: 7770
49 changes: 14 additions & 35 deletions amostra/ignition.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from .server.conf import load_configuration


def start_server(config=None):
def start_server(config=None, testing=False):
"""
Amostra service startup script.
Returns tornado event loop provided configuration.
Expand All @@ -27,58 +27,37 @@ def start_server(config=None):
"""
if not config:
config = {k: v for k, v in load_configuration('amostra', 'AMST',
['mongo_host',
'mongo_port', 'timezone',
['mongo_uri',
'timezone',
'database',
'service_port'],
allow_missing=True).items() if v is not None}
parser = argparse.ArgumentParser()
parser.add_argument('--database', dest='database', type=str,
help='name of database to use')
parser.add_argument('--mongo-host',
dest='mongo_host', type=str,
help='host to use')
parser.add_argument('--mongo_uri',
dest='mongo_uri', type=str,
help='URI for connecting to central MongoDB')
parser.add_argument('--timezone', dest='timezone', type=str,
help='Local timezone')
parser.add_argument('--mongo-port', dest='mongo_port', type=int,
help='port to use to talk to mongo')
parser.add_argument('--service-port', dest='service_port', type=int,
parser.add_argument('--service_port', dest='service_port', type=int,
help='port listen to for clients')
parser.add_argument('--no-auth', dest='auth', action='store_false')
parser.add_argument('--auth', dest='auth', action='store_true')
parser.set_defaults(auth=False)
parser.add_argument('--mongo-user', dest='mongo_user', type=str,
help='Mongo username')
parser.add_argument('--mongo-pwd', dest='mongo_pwd', type=str,
help='Mongo password')
args = parser.parse_args()
print(args)
if args.database is not None:
config['database'] = args.database
if args.mongo_host is not None:
config['mongo_host'] = args.mongo_host
if args.mongo_uri is not None:
config['mongo_uri'] = args.mongo_uri
elif not config["mongo_uri"]:
raise KeyError('mongo_uri must be defined')
if args.timezone is not None:
config['timezone'] = args.timezone
if args.mongo_port is not None:
config['mongo_port'] = args.mongo_port
service_port = args.service_port
if service_port is None:
service_port = 7770
if args.auth:
if args.mongo_user and args.mongo_pwd:
config['mongo_user'] = args.mongo_user
config['mongo_pwd'] = args.mongo_pwd
else:
raise KeyError('--mongo-user and --mongo-pwd required with auth')
else:
config['mongo_user'] = None
config['mongo_pwd'] = None
config['service_port'] = service_port
print(args)
db = db_connect(database=config['database'],
mongo_host=config['mongo_host'],
mongo_port=config['mongo_port'],
mongo_user=config['mongo_user'],
mongo_pwd=config['mongo_pwd'],
auth=args.auth)
mongo_uri=config['mongo_uri'], testing=testing)
application = tornado.web.Application([
(r'/sample', SampleReferenceHandler),
(r'/request', RequestReferenceHandler),
Expand Down
63 changes: 16 additions & 47 deletions amostra/server/engine.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,7 @@
from .utils import compose_err_msg


def db_connect(database, mongo_host, mongo_port, mongo_user=None,
mongo_pwd=None, auth=False):
def db_connect(database, mongo_uri, testing=False):
"""Helper function to deal with stateful connections to MongoDB
Connection established lazily. Connects to the database on request.
Same connection pool is used for all clients per recommended by
Expand All @@ -21,55 +20,25 @@ def db_connect(database, mongo_host, mongo_port, mongo_user=None,
----------
database: str
The name of database pymongo creates and/or connects
host: str
Name/address of the server that mongo daemon lives
port: int
Port num of the server
mongo_uri: str
Name/address of the server with username and password that mongo daemon lives
Returns pymongo.database.Database
-------
Async server object which comes in handy as server has to juggle
multiple clients and makes no difference for a single client compared
to pymongo
"""
if auth:
uri = 'mongodb://{0}:{1}@{2}:{3}/'.format(mongo_user,
mongo_pwd,
mongo_host,
mongo_port)
client = pymongo.MongoClient(uri)
if testing:
import mongomock
client = mongomock.MongoClient(mongo_uri)
else:
try:
client = pymongo.MongoClient(host=mongo_host, port=mongo_port)
except pymongo.errors.ConnectionFailure:
client = pymongo.MongoClient(mongo_uri)
# Proactively check that the connection to server is working.
client.server_info()
except (pymongo.errors.ConnectionFailure, pymongo.errors.ServerSelectionTimeoutError):
raise utils.AmostraException("Unable to connect to MongoDB server...")
database = client[database]
try:
database.sample.create_index([('uid', DESCENDING)],
unique=True, background=True)
database.sample.create_index([('time', DESCENDING),
('name', DESCENDING)],
unique=False, background=True)
database.sample.create_index([('container', DESCENDING)],
unique=False, sparse=True)
database.sample.create_index([('uid', DESCENDING)],
unique=True, background=True)
database.sample.create_index([('time', DESCENDING)],
unique=False, background=True)
database.request.create_index([('uid', DESCENDING)],
unique=True, background=True)
database.request.create_index([('time', DESCENDING)],
unique=False, background=True)
database.request.create_index([('sample', DESCENDING)],
unique=False, background=True, sparse=True)
database.container.create_index([('uid', DESCENDING)],
unique=True, background=True)
database.container.create_index([('time', DESCENDING)],
unique=False, background=True)
database.container.create_index([('container', DESCENDING)],
unique=False, background=True,
sparse=True)
except PyMongoError:
raise compose_err_msg(500, 'Not connected to Mongo daemon')
return database


Expand Down Expand Up @@ -154,7 +123,7 @@ def post(self):
"Invalid schema on document(s)",
d)
uids.append(d['uid'])
res = database.sample.insert(d)
res = database.sample.insert_one(d)
elif isinstance(data, dict):
data = utils.default_timeuid(data)
try:
Expand All @@ -165,7 +134,7 @@ def post(self):
"Invalid schema on document(s)",
data)
uids.append(data['uid'])
res = database.sample.insert(data)
res = database.sample.insert_one(data)
if not res:
raise compose_err_msg(500,
'SampleHandler expects list or dict')
Expand Down Expand Up @@ -231,7 +200,7 @@ def post(self):
"Invalid schema on document(s)",
d)
try:
database.request.insert(d)
database.request.insert_one(d)
uids.append(d['uid'])
except pymongo.errors.PyMongoError:
raise compose_err_msg(500,
Expand All @@ -247,7 +216,7 @@ def post(self):
"Invalid schema on document(s)",
data)
try:
database.request.insert(data)
database.request.insert_one(data)
uids.append(data['uid'])
except pymongo.errors.PyMongoError:
raise compose_err_msg(500,
Expand Down Expand Up @@ -335,7 +304,7 @@ def post(self):
raise compose_err_msg(400,
"Invalid schema on document(s)", d)
uids.append(d['uid'])
res = database.container.insert(d)
res = database.container.insert_one(d)
elif isinstance(data, dict):
data = utils.default_timeuid(data)
try:
Expand All @@ -345,7 +314,7 @@ def post(self):
raise compose_err_msg(400,
"Invalid schema on document(s)", data)
uids.append(data['uid'])
res = database.container.insert(data)
res = database.container.insert_one(data)
if not res:
raise compose_err_msg(500,
'SampleHandler expects list or dict')
Expand Down
4 changes: 4 additions & 0 deletions amostra/startup.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
from .ignition import start_server

if __name__ == "__main__":
start_server()
Loading

0 comments on commit 3c5b566

Please sign in to comment.