From c52aa462cb290083a7423e0bccb8289a21d00a61 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Wed, 6 Jan 2021 13:08:25 +0100 Subject: [PATCH 1/7] use osx sysroot --- cget/cmake/autotools.cmake | 12 +++++++++--- cget/cmake/boost.cmake | 12 +++++++++--- cget/cmake/make.cmake | 12 +++++++++--- cget/cmake/meson.cmake | 12 +++++++++--- tools/preamble.cmake | 12 +++++++++--- 5 files changed, 45 insertions(+), 15 deletions(-) diff --git a/cget/cmake/autotools.cmake b/cget/cmake/autotools.cmake index 5dc9417..3701785 100644 --- a/cget/cmake/autotools.cmake +++ b/cget/cmake/autotools.cmake @@ -103,6 +103,15 @@ macro(preamble PREFIX) if(CMAKE_POSITION_INDEPENDENT_CODE AND NOT WIN32) set(${PREFIX}_PIC_FLAG "-fPIC") endif() + + set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") + set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") + if (APPLE) + set(${PREFIX}_C_FLAGS "${${PREFIX}_C_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + set(${PREFIX}_CXX_FLAGS "${${PREFIX}_CXX_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + set(${PREFIX}_LINK_FLAGS "${${PREFIX}_LINK_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + endif (APPLE) + get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) if(BUILD_SHARED_LIBS) string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_SHARED_LINKER_FLAGS}") @@ -111,9 +120,6 @@ macro(preamble PREFIX) endif() get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) - set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") - set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") - foreach(LANG C CXX) foreach(DIR ${CMAKE_${LANG}_STANDARD_INCLUDE_DIRECTORIES}) if(MSVC) diff --git a/cget/cmake/boost.cmake b/cget/cmake/boost.cmake index 3f8a49a..87127ec 100644 --- a/cget/cmake/boost.cmake +++ b/cget/cmake/boost.cmake @@ -96,6 +96,15 @@ macro(preamble PREFIX) if(CMAKE_POSITION_INDEPENDENT_CODE AND NOT WIN32) set(${PREFIX}_PIC_FLAG "-fPIC") endif() + + set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") + set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") + if (APPLE) + set(${PREFIX}_C_FLAGS "${${PREFIX}_C_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + set(${PREFIX}_CXX_FLAGS "${${PREFIX}_CXX_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + set(${PREFIX}_LINK_FLAGS "${${PREFIX}_LINK_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + endif (APPLE) + get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) if(BUILD_SHARED_LIBS) string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_SHARED_LINKER_FLAGS}") @@ -104,9 +113,6 @@ macro(preamble PREFIX) endif() get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) - set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") - set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") - foreach(LANG C CXX) foreach(DIR ${CMAKE_${LANG}_STANDARD_INCLUDE_DIRECTORIES}) if(MSVC) diff --git a/cget/cmake/make.cmake b/cget/cmake/make.cmake index 40353aa..7d5f495 100644 --- a/cget/cmake/make.cmake +++ b/cget/cmake/make.cmake @@ -101,6 +101,15 @@ macro(preamble PREFIX) if(CMAKE_POSITION_INDEPENDENT_CODE AND NOT WIN32) set(${PREFIX}_PIC_FLAG "-fPIC") endif() + + set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") + set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") + if (APPLE) + set(${PREFIX}_C_FLAGS "${${PREFIX}_C_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + set(${PREFIX}_CXX_FLAGS "${${PREFIX}_CXX_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + set(${PREFIX}_LINK_FLAGS "${${PREFIX}_LINK_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + endif (APPLE) + get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) if(BUILD_SHARED_LIBS) string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_SHARED_LINKER_FLAGS}") @@ -109,9 +118,6 @@ macro(preamble PREFIX) endif() get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) - set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") - set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") - foreach(LANG C CXX) foreach(DIR ${CMAKE_${LANG}_STANDARD_INCLUDE_DIRECTORIES}) if(MSVC) diff --git a/cget/cmake/meson.cmake b/cget/cmake/meson.cmake index f78c1b7..73e4601 100644 --- a/cget/cmake/meson.cmake +++ b/cget/cmake/meson.cmake @@ -130,6 +130,15 @@ macro(preamble PREFIX) if(CMAKE_POSITION_INDEPENDENT_CODE AND NOT WIN32) set(${PREFIX}_PIC_FLAG "-fPIC") endif() + + set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") + set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") + if (APPLE) + set(${PREFIX}_C_FLAGS "${${PREFIX}_C_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + set(${PREFIX}_CXX_FLAGS "${${PREFIX}_CXX_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + set(${PREFIX}_LINK_FLAGS "${${PREFIX}_LINK_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + endif (APPLE) + get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) if(BUILD_SHARED_LIBS) string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_SHARED_LINKER_FLAGS}") @@ -138,9 +147,6 @@ macro(preamble PREFIX) endif() get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) - set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") - set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") - foreach(LANG C CXX) foreach(DIR ${CMAKE_${LANG}_STANDARD_INCLUDE_DIRECTORIES}) if(MSVC) diff --git a/tools/preamble.cmake b/tools/preamble.cmake index d95ca55..ae1fed6 100644 --- a/tools/preamble.cmake +++ b/tools/preamble.cmake @@ -86,6 +86,15 @@ macro(preamble PREFIX) if(CMAKE_POSITION_INDEPENDENT_CODE AND NOT WIN32) set(${PREFIX}_PIC_FLAG "-fPIC") endif() + + set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") + set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") + if (APPLE) + set(${PREFIX}_C_FLAGS "${${PREFIX}_C_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + set(${PREFIX}_CXX_FLAGS "${${PREFIX}_CXX_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + set(${PREFIX}_LINK_FLAGS "${${PREFIX}_LINK_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}") + endif (APPLE) + get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) if(BUILD_SHARED_LIBS) string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_SHARED_LINKER_FLAGS}") @@ -94,9 +103,6 @@ macro(preamble PREFIX) endif() get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS) - set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") - set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}") - foreach(LANG C CXX) foreach(DIR ${CMAKE_${LANG}_STANDARD_INCLUDE_DIRECTORIES}) if(MSVC) From 8ab8918e2365b5e5aaa9495ef1c8eebb8d84f2e5 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Wed, 6 Jan 2021 14:08:29 +0100 Subject: [PATCH 2/7] use requests instead of urlopen to avoid 403 --- cget/util.py | 40 ++++++++++++++++++---------------------- requirements.txt | 1 + 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/cget/util.py b/cget/util.py index b2aa1b5..f7e2457 100644 --- a/cget/util.py +++ b/cget/util.py @@ -16,7 +16,7 @@ else: import subprocess -from six.moves.urllib import request +import requests def to_bool(value): x = str(value).lower() @@ -205,31 +205,27 @@ def symlink_to(src, dst_dir): os.symlink(src, target) return target -class CGetURLOpener(request.FancyURLopener): - def http_error_default(self, url, fp, errcode, errmsg, headers): - if errcode >= 400: - raise BuildError("Download failed with error {0} for: {1}".format(errcode, url)) - return request.FancyURLopener.http_error_default(self, url, fp, errcode, errmsg, headers) - def download_to(url, download_dir, insecure=False): name = url.split('/')[-1] - file = os.path.join(download_dir, name) + file_name = os.path.join(download_dir, name) click.echo("Downloading {0}".format(url)) - bar_len = 1000 - with click.progressbar(length=bar_len, width=70) as bar: - def hook(count, block_size, total_size): - percent = int(count*block_size*bar_len/total_size) - if percent > 0 and percent < bar_len: - # Hack because we can't set the position - bar.pos = percent - bar.update(0) - context = None - if insecure: context = ssl._create_unverified_context() - CGetURLOpener(context=context).retrieve(url, filename=file, reporthook=hook, data=None) - bar.update(bar_len) - if not os.path.exists(file): + with open(file_name, "wb") as f: + response = requests.get(url, stream=True) + total_length = response.headers.get('content-length') + if total_length is None: # no content length header + f.write(response.content) + else: + dl = 0 + total_length = int(total_length) + for data in response.iter_content(chunk_size=4096): + dl += len(data) + f.write(data) + done = int(50 * dl / total_length) + sys.stdout.write("\r[%s%s]" % ('=' * done, ' ' * (50-done)) ) + sys.stdout.flush() + if not os.path.exists(file_name): raise BuildError("Download failed for: {0}".format(url)) - return file + return file_name def transfer_to(f, dst, copy=False): if USE_SYMLINKS and not copy: return symlink_to(f, dst) diff --git a/requirements.txt b/requirements.txt index 1df73e3..2e2a871 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ click>=6.6 # PyYAML six>=1.10 +requests From 7c11f74b176d9b05556db1193e3e963f607c34f4 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Wed, 6 Jan 2021 14:08:29 +0100 Subject: [PATCH 3/7] use requests instead of urlopen to avoid 403 --- cget/util.py | 40 ++++++++++++++++++---------------------- requirements.txt | 1 + 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/cget/util.py b/cget/util.py index b2aa1b5..f7e2457 100644 --- a/cget/util.py +++ b/cget/util.py @@ -16,7 +16,7 @@ else: import subprocess -from six.moves.urllib import request +import requests def to_bool(value): x = str(value).lower() @@ -205,31 +205,27 @@ def symlink_to(src, dst_dir): os.symlink(src, target) return target -class CGetURLOpener(request.FancyURLopener): - def http_error_default(self, url, fp, errcode, errmsg, headers): - if errcode >= 400: - raise BuildError("Download failed with error {0} for: {1}".format(errcode, url)) - return request.FancyURLopener.http_error_default(self, url, fp, errcode, errmsg, headers) - def download_to(url, download_dir, insecure=False): name = url.split('/')[-1] - file = os.path.join(download_dir, name) + file_name = os.path.join(download_dir, name) click.echo("Downloading {0}".format(url)) - bar_len = 1000 - with click.progressbar(length=bar_len, width=70) as bar: - def hook(count, block_size, total_size): - percent = int(count*block_size*bar_len/total_size) - if percent > 0 and percent < bar_len: - # Hack because we can't set the position - bar.pos = percent - bar.update(0) - context = None - if insecure: context = ssl._create_unverified_context() - CGetURLOpener(context=context).retrieve(url, filename=file, reporthook=hook, data=None) - bar.update(bar_len) - if not os.path.exists(file): + with open(file_name, "wb") as f: + response = requests.get(url, stream=True) + total_length = response.headers.get('content-length') + if total_length is None: # no content length header + f.write(response.content) + else: + dl = 0 + total_length = int(total_length) + for data in response.iter_content(chunk_size=4096): + dl += len(data) + f.write(data) + done = int(50 * dl / total_length) + sys.stdout.write("\r[%s%s]" % ('=' * done, ' ' * (50-done)) ) + sys.stdout.flush() + if not os.path.exists(file_name): raise BuildError("Download failed for: {0}".format(url)) - return file + return file_name def transfer_to(f, dst, copy=False): if USE_SYMLINKS and not copy: return symlink_to(f, dst) diff --git a/requirements.txt b/requirements.txt index 1df73e3..2e2a871 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ click>=6.6 # PyYAML six>=1.10 +requests From 0051c345a7c2e10848842704d62b64cdfb4cc670 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Thu, 7 Jan 2021 07:04:34 +0100 Subject: [PATCH 4/7] drop dead import --- cget/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cget/util.py b/cget/util.py index f7e2457..0812320 100644 --- a/cget/util.py +++ b/cget/util.py @@ -1,4 +1,4 @@ -import click, os, sys, shutil, json, six, hashlib, ssl +import click, os, sys, shutil, json, six, hashlib if sys.version_info[0] < 3: try: From 70e05749392a04658f8c7d1f30740dad5b110731 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Thu, 7 Jan 2021 07:30:33 +0100 Subject: [PATCH 5/7] drop ssl import --- cget/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cget/util.py b/cget/util.py index f7e2457..0812320 100644 --- a/cget/util.py +++ b/cget/util.py @@ -1,4 +1,4 @@ -import click, os, sys, shutil, json, six, hashlib, ssl +import click, os, sys, shutil, json, six, hashlib if sys.version_info[0] < 3: try: From fa9af070cffcc5ec29fe358164ecd9eb38e2f1c6 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Thu, 7 Jan 2021 07:30:42 +0100 Subject: [PATCH 6/7] use click progressbar --- cget/util.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/cget/util.py b/cget/util.py index 0812320..7acf966 100644 --- a/cget/util.py +++ b/cget/util.py @@ -215,14 +215,13 @@ def download_to(url, download_dir, insecure=False): if total_length is None: # no content length header f.write(response.content) else: - dl = 0 total_length = int(total_length) - for data in response.iter_content(chunk_size=4096): - dl += len(data) - f.write(data) - done = int(50 * dl / total_length) - sys.stdout.write("\r[%s%s]" % ('=' * done, ' ' * (50-done)) ) - sys.stdout.flush() + with click.progressbar(length=total_length, width=70) as bar: + for data in response.iter_content(chunk_size=4096): + f.write(data) + bar.pos += len(data) + bar.update(0) + bar.update(total_length) if not os.path.exists(file_name): raise BuildError("Download failed for: {0}".format(url)) return file_name From 858acb5c810ad02157032c6047ef984e8793cec4 Mon Sep 17 00:00:00 2001 From: Daniel Oberhoff Date: Mon, 11 Jan 2021 09:59:02 +0100 Subject: [PATCH 7/7] specify requests version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2e2a871..a8f359b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ click>=6.6 # PyYAML six>=1.10 -requests +requests>=2.25.1