Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

use requests instead of urlopen to avoid 403 #84

Open
wants to merge 8 commits into
base: master
Choose a base branch
from
Open

use requests instead of urlopen to avoid 403 #84

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
c52aa46
use osx sysroot
maddanio Jan 6, 2021
8ab8918
use requests instead of urlopen to avoid 403
maddanio Jan 6, 2021
7c11f74
use requests instead of urlopen to avoid 403
maddanio Jan 6, 2021
0051c34
drop dead import
maddanio Jan 7, 2021
70e0574
drop ssl import
maddanio Jan 7, 2021
fa9af07
use click progressbar
maddanio Jan 7, 2021
a619e89
Merge branch 'master' into nourlopen
maddanio Jan 7, 2021
858acb5
specify requests version
maddanio Jan 11, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions cget/cmake/autotools.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,15 @@ macro(preamble PREFIX)
if(CMAKE_POSITION_INDEPENDENT_CODE AND NOT WIN32)
set(${PREFIX}_PIC_FLAG "-fPIC")
endif()

set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}")
set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}")
if (APPLE)
set(${PREFIX}_C_FLAGS "${${PREFIX}_C_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}")
set(${PREFIX}_CXX_FLAGS "${${PREFIX}_CXX_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}")
set(${PREFIX}_LINK_FLAGS "${${PREFIX}_LINK_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}")
endif (APPLE)

get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS)
if(BUILD_SHARED_LIBS)
string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_SHARED_LINKER_FLAGS}")
Expand All @@ -111,9 +120,6 @@ macro(preamble PREFIX)
endif()
get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS)

set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}")
set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}")

foreach(LANG C CXX)
foreach(DIR ${CMAKE_${LANG}_STANDARD_INCLUDE_DIRECTORIES})
if(MSVC)
Expand Down
12 changes: 9 additions & 3 deletions cget/cmake/boost.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,15 @@ macro(preamble PREFIX)
if(CMAKE_POSITION_INDEPENDENT_CODE AND NOT WIN32)
set(${PREFIX}_PIC_FLAG "-fPIC")
endif()

set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}")
set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}")
if (APPLE)
set(${PREFIX}_C_FLAGS "${${PREFIX}_C_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}")
set(${PREFIX}_CXX_FLAGS "${${PREFIX}_CXX_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}")
set(${PREFIX}_LINK_FLAGS "${${PREFIX}_LINK_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}")
endif (APPLE)

get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS)
if(BUILD_SHARED_LIBS)
string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_SHARED_LINKER_FLAGS}")
Expand All @@ -104,9 +113,6 @@ macro(preamble PREFIX)
endif()
get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS)

set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}")
set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}")

foreach(LANG C CXX)
foreach(DIR ${CMAKE_${LANG}_STANDARD_INCLUDE_DIRECTORIES})
if(MSVC)
Expand Down
12 changes: 9 additions & 3 deletions cget/cmake/make.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,15 @@ macro(preamble PREFIX)
if(CMAKE_POSITION_INDEPENDENT_CODE AND NOT WIN32)
set(${PREFIX}_PIC_FLAG "-fPIC")
endif()

set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}")
set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}")
if (APPLE)
set(${PREFIX}_C_FLAGS "${${PREFIX}_C_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}")
set(${PREFIX}_CXX_FLAGS "${${PREFIX}_CXX_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}")
set(${PREFIX}_LINK_FLAGS "${${PREFIX}_LINK_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}")
endif (APPLE)

get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS)
if(BUILD_SHARED_LIBS)
string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_SHARED_LINKER_FLAGS}")
Expand All @@ -109,9 +118,6 @@ macro(preamble PREFIX)
endif()
get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS)

set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}")
set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}")

foreach(LANG C CXX)
foreach(DIR ${CMAKE_${LANG}_STANDARD_INCLUDE_DIRECTORIES})
if(MSVC)
Expand Down
12 changes: 9 additions & 3 deletions cget/cmake/meson.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,15 @@ macro(preamble PREFIX)
if(CMAKE_POSITION_INDEPENDENT_CODE AND NOT WIN32)
set(${PREFIX}_PIC_FLAG "-fPIC")
endif()

set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}")
set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}")
if (APPLE)
set(${PREFIX}_C_FLAGS "${${PREFIX}_C_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}")
set(${PREFIX}_CXX_FLAGS "${${PREFIX}_CXX_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}")
set(${PREFIX}_LINK_FLAGS "${${PREFIX}_LINK_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}")
endif (APPLE)

get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS)
if(BUILD_SHARED_LIBS)
string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_SHARED_LINKER_FLAGS}")
Expand All @@ -138,9 +147,6 @@ macro(preamble PREFIX)
endif()
get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS)

set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}")
set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}")

foreach(LANG C CXX)
foreach(DIR ${CMAKE_${LANG}_STANDARD_INCLUDE_DIRECTORIES})
if(MSVC)
Expand Down
41 changes: 18 additions & 23 deletions cget/util.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import click, os, sys, shutil, json, six, hashlib, ssl
import click, os, sys, shutil, json, six, hashlib

if sys.version_info[0] < 3:
try:
Expand All @@ -16,7 +16,7 @@
else:
import subprocess

from six.moves.urllib import request
import requests

def to_bool(value):
x = str(value).lower()
Expand Down Expand Up @@ -205,31 +205,26 @@ def symlink_to(src, dst_dir):
os.symlink(src, target)
return target

class CGetURLOpener(request.FancyURLopener):
def http_error_default(self, url, fp, errcode, errmsg, headers):
if errcode >= 400:
raise BuildError("Download failed with error {0} for: {1}".format(errcode, url))
return request.FancyURLopener.http_error_default(self, url, fp, errcode, errmsg, headers)

def download_to(url, download_dir, insecure=False):
name = url.split('/')[-1]
file = os.path.join(download_dir, name)
file_name = os.path.join(download_dir, name)
click.echo("Downloading {0}".format(url))
bar_len = 1000
with click.progressbar(length=bar_len, width=70) as bar:
def hook(count, block_size, total_size):
percent = int(count*block_size*bar_len/total_size)
if percent > 0 and percent < bar_len:
# Hack because we can't set the position
bar.pos = percent
bar.update(0)
context = None
if insecure: context = ssl._create_unverified_context()
CGetURLOpener(context=context).retrieve(url, filename=file, reporthook=hook, data=None)
bar.update(bar_len)
if not os.path.exists(file):
with open(file_name, "wb") as f:
response = requests.get(url, stream=True)
total_length = response.headers.get('content-length')
if total_length is None: # no content length header
f.write(response.content)
else:
total_length = int(total_length)
with click.progressbar(length=total_length, width=70) as bar:
for data in response.iter_content(chunk_size=4096):
f.write(data)
bar.pos += len(data)
bar.update(0)
bar.update(total_length)
if not os.path.exists(file_name):
raise BuildError("Download failed for: {0}".format(url))
return file
return file_name

def transfer_to(f, dst, copy=False):
if USE_SYMLINKS and not copy: return symlink_to(f, dst)
Expand Down
1 change: 1 addition & 0 deletions requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
click>=6.6
# PyYAML
six>=1.10
requests>=2.25.1
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This probably needs to be version 2.21.0 as this version doesnt seem to work with python 3.4.

12 changes: 9 additions & 3 deletions tools/preamble.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,15 @@ macro(preamble PREFIX)
if(CMAKE_POSITION_INDEPENDENT_CODE AND NOT WIN32)
set(${PREFIX}_PIC_FLAG "-fPIC")
endif()

set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}")
set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}")
if (APPLE)
set(${PREFIX}_C_FLAGS "${${PREFIX}_C_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}")
set(${PREFIX}_CXX_FLAGS "${${PREFIX}_CXX_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}")
set(${PREFIX}_LINK_FLAGS "${${PREFIX}_LINK_FLAGS} -isysroot ${CMAKE_OSX_SYSROOT}")
endif (APPLE)

get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS)
if(BUILD_SHARED_LIBS)
string(APPEND ${PREFIX}_LINK_FLAGS " ${CMAKE_SHARED_LINKER_FLAGS}")
Expand All @@ -94,9 +103,6 @@ macro(preamble PREFIX)
endif()
get_property_list(${PREFIX}_LINK_FLAGS LINK_FLAGS)

set(${PREFIX}_C_FLAGS "${CMAKE_C_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}")
set(${PREFIX}_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${${PREFIX}_COMPILE_FLAGS} ${${PREFIX}_PIC_FLAG}")

foreach(LANG C CXX)
foreach(DIR ${CMAKE_${LANG}_STANDARD_INCLUDE_DIRECTORIES})
if(MSVC)
Expand Down