Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rename blacklist #11

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 13 additions & 13 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -89,27 +89,27 @@ WARNING: Invoking this command will pull ~10GB and take at least an hour

## More Details

### blacklist/whitelist configuration
### denylist/allowlist configuration

example-conf.yaml:

```yaml
blacklist:
denylist:
- license: "*agpl*"
- license: None
- license: ""

whitelist:
allowlist:
- name: system
```

`blacklist` removes package(s) that match the condition(s) listed from the
`denylist` removes package(s) that match the condition(s) listed from the
upstream repodata.

`whitelist` re-includes any package(s) from blacklist that match the
whitelist conditions.
`allowlist` re-includes any package(s) from denylist that match the
allowlist conditions.

blacklist and whitelist both take lists of dictionaries. The keys in the
denylist and allowlist both take lists of dictionaries. The keys in the
dictionary need to be values in the `repodata.json` metadata. The values are
(unix) globs to match on. Go here for the full repodata of the upstream
"defaults" channel:
Expand Down Expand Up @@ -144,28 +144,28 @@ information.
#### Common usage patterns
##### Mirror **only** one specific package
If you wanted to match exactly the botocore package listed above with your
config, then you could use the following configuration to first blacklist
config, then you could use the following configuration to first denylist
**all** packages and then include just the botocore packages:

```yaml
blacklist:
denylist:
- name: "*"
whitelist:
allowlist:
- name: botocore
version: 1.4.10
build: py34_0
```
##### Mirror everything but agpl licenses
```yaml
blacklist:
denylist:
- license: "*agpl*"
```

##### Mirror only python 3 packages
```yaml
blacklist:
denylist:
- name: "*"
whitelist:
allowlist:
- build: "*py3*"
```

Expand Down
93 changes: 59 additions & 34 deletions conda_mirror/conda_mirror.py
Original file line number Diff line number Diff line change
Expand Up @@ -300,7 +300,32 @@ def _parse_and_format_args():
setattr(args, a.dest, config_dict.get(a.dest))

blacklist = config_dict.get('blacklist')
denylist = config_dict.get('denylist')
whitelist = config_dict.get('whitelist')
allowlist = config_dict.get('allowlist')
if blacklist and denylist:
# Would be odd to be here, but might as well be nice to the users
logger.warning(
"Please remove blacklist from your config. "
"This will stop working in the next minor release")
if whitelist and allowlist:
# Would be odd to be here, but might as well be nice to the users
logger.warning(
"Please remove whitelist from your config. "
"This will stop working in the next minor release")
if blacklist and not denylist:
logger.warning(
"Please rename blacklist to denylist in your config. "
"This will stop working in the next minor release")
denylist = blacklist
if whitelist and not allowlist:
logger.warning(
"Please rename whitelist to allowlist in your config. "
"This will stop working in the next minor release")
allowlist = whitelist

del blacklist
del whitelist

for required in ('target_directory', 'platform', 'upstream_channel'):
if (not getattr(args, required)):
Expand Down Expand Up @@ -333,8 +358,8 @@ def pdb_hook(exctype, value, traceback):
'temp_directory': args.temp_directory,
'platform': args.platform,
'num_threads': args.num_threads,
'blacklist': blacklist,
'whitelist': whitelist,
'denylist': denylist,
'allowlist': allowlist,
'dry_run': args.dry_run,
'no_validate_target': args.no_validate_target,
'minimum_free_space': args.minimum_free_space,
Expand Down Expand Up @@ -656,7 +681,7 @@ def _validate_or_remove_package(args):


def main(upstream_channel, target_directory, temp_directory, platform,
blacklist=None, whitelist=None, num_threads=1, dry_run=False,
denylist=None, allowlist=None, num_threads=1, dry_run=False,
no_validate_target=False, minimum_free_space=0, proxies=None,
ssl_verify=None, max_retries=100):
"""
Expand All @@ -679,12 +704,12 @@ def main(upstream_channel, target_directory, temp_directory, platform,
The platform that you wish to mirror for. Common options are
'linux-64', 'osx-64', 'win-64' and 'win-32'. Any platform is valid as
long as the url resolves.
blacklist : iterable of tuples, optional
The values of blacklist should be (key, glob) where key is one of the
denylist : iterable of tuples, optional
The values of denylist should be (key, glob) where key is one of the
keys in the repodata['packages'] dicts and glob is a thing to match
on. Note that all comparisons will be laundered through lowercasing.
whitelist : iterable of tuples, optional
The values of blacklist should be (key, glob) where key is one of the
allowlist : iterable of tuples, optional
The values of denylist should be (key, glob) where key is one of the
keys in the repodata['packages'] dicts and glob is a thing to match
on. Note that all comparisons will be laundered through lowercasing.
num_threads : int, optional
Expand Down Expand Up @@ -747,9 +772,9 @@ def main(upstream_channel, target_directory, temp_directory, platform,
'version': '8.5.18'}
"""
# Steps:
# 1. figure out blacklisted packages
# 2. un-blacklist packages that are actually whitelisted
# 3. remove blacklisted packages
# 1. figure out denylisted packages
# 2. un-denylist packages that are actually allowlisted
# 3. remove denylisted packages
# 4. figure out final list of packages to mirror
# 5. mirror new packages to temp dir
# 6. validate new packages
Expand All @@ -760,7 +785,7 @@ def main(upstream_channel, target_directory, temp_directory, platform,
'validating-existing': set(),
'validating-new': set(),
'downloaded': set(),
'blacklisted': set(),
'denylisted': set(),
'to-mirror': set()
}
# Implementation:
Expand All @@ -777,43 +802,43 @@ def main(upstream_channel, target_directory, temp_directory, platform,
# package_directory=local_directory,
# num_threads=num_threads)

# 2. figure out blacklisted packages
blacklist_packages = {}
whitelist_packages = {}
# match blacklist conditions
if blacklist:
blacklist_packages = {}
for blist in blacklist:
logger.debug('blacklist item: %s', blist)
# 2. figure out denylisted packages
denylist_packages = {}
allowlist_packages = {}
# match denylist conditions
if denylist:
denylist_packages = {}
for blist in denylist:
logger.debug('denylist item: %s', blist)
matched_packages = _match(packages, blist)
logger.debug(pformat(list(matched_packages.keys())))
blacklist_packages.update(matched_packages)
denylist_packages.update(matched_packages)

# 3. un-blacklist packages that are actually whitelisted
# match whitelist on blacklist
if whitelist:
whitelist_packages = {}
for wlist in whitelist:
# 3. un-denylist packages that are actually allowlisted
# match allowlist on denylist
if allowlist:
allowlist_packages = {}
for wlist in allowlist:
matched_packages = _match(packages, wlist)
whitelist_packages.update(matched_packages)
# make final mirror list of not-blacklist + whitelist
true_blacklist = set(blacklist_packages.keys()) - set(
whitelist_packages.keys())
summary['blacklisted'].update(true_blacklist)
allowlist_packages.update(matched_packages)
# make final mirror list of not-denylist + allowlist
true_denylist = set(denylist_packages.keys()) - set(
allowlist_packages.keys())
summary['denylisted'].update(true_denylist)

logger.info("BLACKLISTED PACKAGES")
logger.info(pformat(true_blacklist))
logger.info("DENIED PACKAGES")
logger.info(pformat(true_denylist))

# Get a list of all packages in the local mirror
if dry_run:
local_packages = _list_conda_packages(local_directory)
packages_slated_for_removal = [
pkg_name for pkg_name in local_packages if pkg_name in summary['blacklisted']
pkg_name for pkg_name in local_packages if pkg_name in summary['denylisted']
]
logger.info("PACKAGES TO BE REMOVED")
logger.info(pformat(packages_slated_for_removal))

possible_packages_to_mirror = set(packages.keys()) - true_blacklist
possible_packages_to_mirror = set(packages.keys()) - true_denylist

# 4. Validate all local packages
# construct the desired package repodata
Expand Down
4 changes: 2 additions & 2 deletions example-conf.yaml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
blacklist:
denylist:
- license: "*agpl*"
- license: None
- license: ""
- name: "*"
whitelist:
allowlist:
- name: system
12 changes: 6 additions & 6 deletions test/test_conda_mirror.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,9 @@ def test_cli(tmpdir, channel, platform, repodata, num_threads):
f1 = tmpdir.mkdir('conf').join('conf.yaml')

f1.write('''
blacklist:
denylist:
- name: "*"
whitelist:
allowlist:
- name: {}
version: {}'''.format(packages[smallest_package]['name'],
packages[smallest_package]['version']))
Expand Down Expand Up @@ -138,8 +138,8 @@ def test_main(tmpdir, repodata):
target_directory=target_directory.strpath,
temp_directory=temp_directory.strpath,
platform=platform,
blacklist=[{'name': '*'}],
whitelist=[{'name': packages[smallest_package]['name'],
denylist=[{'name': '*'}],
allowlist=[{'name': packages[smallest_package]['name'],
'version': packages[smallest_package]['version']}])

assert len(ret['validating-existing']) == 0, "There should be no already-downloaded packages"
Expand All @@ -152,8 +152,8 @@ def test_main(tmpdir, repodata):
target_directory=target_directory.strpath,
temp_directory=temp_directory.strpath,
platform=platform,
blacklist=[{'name': '*'}],
whitelist=[{'name': packages[next_smallest_package]['name'],
denylist=[{'name': '*'}],
allowlist=[{'name': packages[next_smallest_package]['name'],
'version': packages[next_smallest_package]['version']}])

msg = "We should have %s packages downloaded now" % previously_downloaded_packages
Expand Down