Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ip address geolocation lookup #3887

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

ip address geolocation lookup #3887

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
207e977
ip address geolocation lookup
aptxx Aug 3, 2024
7906816
Merge branch 'master' into geolocation
aptxx Feb 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 14 additions & 4 deletions config/account.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ type Account struct {
DefaultBidLimit int `mapstructure:"default_bid_limit" json:"default_bid_limit"`
BidAdjustments *openrtb_ext.ExtRequestPrebidBidAdjustments `mapstructure:"bidadjustments" json:"bidadjustments"`
Privacy AccountPrivacy `mapstructure:"privacy" json:"privacy"`
GeoLocation AccountGeoLocation `mapstructure:"geolocation" json:"geolocation"`
}

// CookieSync represents the account-level defaults for the cookie sync endpoint.
Expand Down Expand Up @@ -156,10 +157,11 @@ type AccountGDPR struct {
Purpose9 AccountGDPRPurpose `mapstructure:"purpose9" json:"purpose9"`
Purpose10 AccountGDPRPurpose `mapstructure:"purpose10" json:"purpose10"`
// Hash table of purpose configs for convenient purpose config lookup
PurposeConfigs map[consentconstants.Purpose]*AccountGDPRPurpose
PurposeOneTreatment AccountGDPRPurposeOneTreatment `mapstructure:"purpose_one_treatment" json:"purpose_one_treatment"`
SpecialFeature1 AccountGDPRSpecialFeature `mapstructure:"special_feature1" json:"special_feature1"`
EEACountries []string `mapstructure:"eea_countries" json:"eea_countries"`
PurposeConfigs map[consentconstants.Purpose]*AccountGDPRPurpose
PurposeOneTreatment AccountGDPRPurposeOneTreatment `mapstructure:"purpose_one_treatment" json:"purpose_one_treatment"`
SpecialFeature1 AccountGDPRSpecialFeature `mapstructure:"special_feature1" json:"special_feature1"`
EEACountries []string `mapstructure:"eea_countries" json:"eea_countries"`
ConsentStringMeansInScope *bool `mapstructure:"consent_string_means_in_scope" json:"consent_string_means_in_scope"`
}

// EnabledForChannelType indicates whether GDPR is turned on at the account level for the specified channel type
Expand Down Expand Up @@ -352,6 +354,14 @@ type CookieDeprecation struct {
TTLSec int `mapstructure:"ttl_sec"`
}

type AccountGeoLocation struct {
Enabled bool `mapstructure:"enabled" json:"enabled,omitempty"`
}

func (g *AccountGeoLocation) IsGeoLocationEnabled() bool {
return g.Enabled
}

// AccountDSA represents DSA configuration
type AccountDSA struct {
Default string `mapstructure:"default" json:"default"`
Expand Down
24 changes: 24 additions & 0 deletions config/account_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1014,3 +1014,27 @@ func TestIPMaskingValidate(t *testing.T) {
})
}
}

func TestGeoLocation(t *testing.T) {
tests := []struct {
geoloc *AccountGeoLocation
expected bool
}{
{
geoloc: &AccountGeoLocation{
Enabled: true,
},
expected: true,
},
{
geoloc: &AccountGeoLocation{
Enabled: false,
},
expected: false,
},
}

for _, test := range tests {
assert.Equal(t, test.expected, test.geoloc.IsGeoLocationEnabled())
}
}
43 changes: 41 additions & 2 deletions config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ type Configuration struct {
Hooks Hooks `mapstructure:"hooks"`
Validations Validations `mapstructure:"validations"`
PriceFloors PriceFloors `mapstructure:"price_floors"`
GeoLocation GeoLocation `mapstructure:"geolocation"`
}

type Admin struct {
Expand Down Expand Up @@ -253,8 +254,9 @@ type GDPR struct {
// If the gdpr flag is unset in a request, but geo.country is set, we will assume GDPR applies if and only
// if the country matches one on this list. If both the GDPR flag and country are not set, we default
// to DefaultValue
EEACountries []string `mapstructure:"eea_countries"`
EEACountriesMap map[string]struct{}
EEACountries []string `mapstructure:"eea_countries"`
EEACountriesMap map[string]struct{}
ConsentStringMeansInScope bool `mapstructure:"consent_string_means_in_scope"`
}

func (cfg *GDPR) validate(v *viper.Viper, errs []error) []error {
Expand Down Expand Up @@ -659,6 +661,27 @@ type DefReqFiles struct {
FileName string `mapstructure:"name"`
}

type GeoLocation struct {
Enabled bool `mapstructure:"enabled"`
Type string `mapstructure:"type"`
Maxmind GeoLocationMaxmind `mapstructure:"maxmind"`
}

type GeoLocationMaxmind struct {
RemoteFileSyncer MaxmindRemoteFileSyncer `mapstructure:"remote_file_syncer"`
}

type MaxmindRemoteFileSyncer struct {
HttpClient HTTPClient `mapstructure:"http_client"`
DownloadURL string `mapstructure:"download_url"`
SaveFilePath string `mapstructure:"save_filepath"`
TmpFilePath string `mapstructure:"tmp_filepath"`
RetryCount int `mapstructure:"retry_count"`
RetryIntervalMillis int `mapstructure:"retry_interval_ms"`
TimeoutMillis int `mapstructure:"timeout_ms"`
UpdateIntervalMillis int `mapstructure:"update_interval_ms"`
}

type Debug struct {
TimeoutNotification TimeoutNotification `mapstructure:"timeout_notification"`
OverrideToken string `mapstructure:"override_token"`
Expand Down Expand Up @@ -1140,6 +1163,7 @@ func SetupViper(v *viper.Viper, filename string, bidderInfos BidderInfos) {
"FIN", "FRA", "GUF", "DEU", "GIB", "GRC", "GLP", "GGY", "HUN", "ISL", "IRL", "IMN", "ITA", "JEY", "LVA",
"LIE", "LTU", "LUX", "MLT", "MTQ", "MYT", "NLD", "NOR", "POL", "PRT", "REU", "ROU", "BLM", "MAF", "SPM",
"SVK", "SVN", "ESP", "SWE", "GBR"})
v.SetDefault("gdpr.consent_string_means_in_scope", false)
v.SetDefault("ccpa.enforce", false)
v.SetDefault("lmt.enforce", true)
v.SetDefault("currency_converter.fetch_url", "https://cdn.jsdelivr.net/gh/prebid/currency-file@1/latest.json")
Expand Down Expand Up @@ -1170,6 +1194,7 @@ func SetupViper(v *viper.Viper, filename string, bidderInfos BidderInfos) {
v.SetDefault("account_defaults.privacy.privacysandbox.topicsdomain", "")
v.SetDefault("account_defaults.privacy.privacysandbox.cookiedeprecation.enabled", false)
v.SetDefault("account_defaults.privacy.privacysandbox.cookiedeprecation.ttl_sec", 604800)
v.SetDefault("account_defaults.geolocation.enabled", false)

v.SetDefault("account_defaults.events_enabled", false)
v.BindEnv("account_defaults.privacy.dsa.default")
Expand All @@ -1187,6 +1212,20 @@ func SetupViper(v *viper.Viper, filename string, bidderInfos BidderInfos) {
v.SetDefault("price_floors.fetcher.http_client.idle_connection_timeout_seconds", 60)
v.SetDefault("price_floors.fetcher.max_retries", 10)

v.SetDefault("geolocation.enabled", false)
v.SetDefault("geolocation.type", "maxmind")
v.SetDefault("geolocation.maxmind.remote_file_syncer.download_url", "https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz")
v.SetDefault("geolocation.maxmind.remote_file_syncer.save_filepath", "/var/tmp/prebid/GeoLite2-City.tar.gz")
v.SetDefault("geolocation.maxmind.remote_file_syncer.tmp_filepath", "/var/tmp/prebid/tmp/GeoLite2-City.tar.gz")
v.SetDefault("geolocation.maxmind.remote_file_syncer.retry_count", 3)
v.SetDefault("geolocation.maxmind.remote_file_syncer.retry_interval_ms", 3000)
v.SetDefault("geolocation.maxmind.remote_file_syncer.timeout_ms", 300000)
v.SetDefault("geolocation.maxmind.remote_file_syncer.update_interval_ms", 0)
v.SetDefault("geolocation.maxmind.remote_file_syncer.http_client.max_connections_per_host", 0)
v.SetDefault("geolocation.maxmind.remote_file_syncer.http_client.max_idle_connections", 40)
v.SetDefault("geolocation.maxmind.remote_file_syncer.http_client.max_idle_connections_per_host", 2)
v.SetDefault("geolocation.maxmind.remote_file_syncer.http_client.idle_connection_timeout_seconds", 60)

v.SetDefault("account_defaults.events_enabled", false)
v.SetDefault("compression.response.enable_gzip", false)
v.SetDefault("compression.request.enable_gzip", false)
Expand Down
53 changes: 53 additions & 0 deletions config/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,9 +209,24 @@ func TestDefaults(t *testing.T) {
cmpStrings(t, "account_defaults.privacy.topicsdomain", "", cfg.AccountDefaults.Privacy.PrivacySandbox.TopicsDomain)
cmpBools(t, "account_defaults.privacy.privacysandbox.cookiedeprecation.enabled", false, cfg.AccountDefaults.Privacy.PrivacySandbox.CookieDeprecation.Enabled)
cmpInts(t, "account_defaults.privacy.privacysandbox.cookiedeprecation.ttl_sec", 604800, cfg.AccountDefaults.Privacy.PrivacySandbox.CookieDeprecation.TTLSec)
cmpBools(t, "account_defaults.geolocation.enabled", false, cfg.AccountDefaults.GeoLocation.Enabled)

cmpBools(t, "account_defaults.events.enabled", false, cfg.AccountDefaults.Events.Enabled)

cmpBools(t, "geolocation.enabled", false, cfg.GeoLocation.Enabled)
cmpStrings(t, "geolocation.type", "maxmind", cfg.GeoLocation.Type)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.http_client.max_connections_per_host", 0, cfg.GeoLocation.Maxmind.RemoteFileSyncer.HttpClient.MaxConnsPerHost)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.http_client.max_idle_connections", 40, cfg.GeoLocation.Maxmind.RemoteFileSyncer.HttpClient.MaxIdleConns)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.http_client.max_idle_connections_per_host", 2, cfg.GeoLocation.Maxmind.RemoteFileSyncer.HttpClient.MaxIdleConnsPerHost)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.http_client.idle_connection_timeout_seconds", 60, cfg.GeoLocation.Maxmind.RemoteFileSyncer.HttpClient.IdleConnTimeout)
cmpStrings(t, "geolocation.maxmind.remote_file_syncer.download_url", "https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz", cfg.GeoLocation.Maxmind.RemoteFileSyncer.DownloadURL)
cmpStrings(t, "geolocation.maxmind.remote_file_syncer.save_filepath", "/var/tmp/prebid/GeoLite2-City.tar.gz", cfg.GeoLocation.Maxmind.RemoteFileSyncer.SaveFilePath)
cmpStrings(t, "geolocation.maxmind.remote_file_syncer.tmp_filepath", "/var/tmp/prebid/tmp/GeoLite2-City.tar.gz", cfg.GeoLocation.Maxmind.RemoteFileSyncer.TmpFilePath)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.retry_count", 3, cfg.GeoLocation.Maxmind.RemoteFileSyncer.RetryCount)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.retry_interval_ms", 3000, cfg.GeoLocation.Maxmind.RemoteFileSyncer.RetryIntervalMillis)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.timeout_ms", 300000, cfg.GeoLocation.Maxmind.RemoteFileSyncer.TimeoutMillis)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.update_interval_ms", 0, cfg.GeoLocation.Maxmind.RemoteFileSyncer.UpdateIntervalMillis)

cmpBools(t, "hooks.enabled", false, cfg.Hooks.Enabled)
cmpStrings(t, "validations.banner_creative_max_size", "skip", cfg.Validations.BannerCreativeMaxSize)
cmpStrings(t, "validations.secure_markup", "skip", cfg.Validations.SecureMarkup)
Expand All @@ -227,6 +242,8 @@ func TestDefaults(t *testing.T) {
cmpInts(t, "account_defaults.privacy.ipv6.anon_keep_bits", 56, cfg.AccountDefaults.Privacy.IPv6Config.AnonKeepBits)
cmpInts(t, "account_defaults.privacy.ipv4.anon_keep_bits", 24, cfg.AccountDefaults.Privacy.IPv4Config.AnonKeepBits)

cmpBools(t, "gdpr.consent_string_means_in_scope", false, cfg.GDPR.ConsentStringMeansInScope)

//Assert purpose VendorExceptionMap hash tables were built correctly
cmpBools(t, "analytics.agma.enabled", false, cfg.Analytics.Agma.Enabled)
cmpStrings(t, "analytics.agma.endpoint.timeout", "2s", cfg.Analytics.Agma.Endpoint.Timeout)
Expand Down Expand Up @@ -350,6 +367,7 @@ gdpr:
default_value: "1"
non_standard_publishers: ["pub1", "pub2"]
eea_countries: ["eea1", "eea2"]
consent_string_means_in_scope: false
tcf2:
purpose1:
enforce_vendors: false
Expand Down Expand Up @@ -494,6 +512,23 @@ price_floors:
max_idle_connections_per_host: 2
idle_connection_timeout_seconds: 10
max_retries: 5
geolocation:
enabled: false
type: maxmind
maxmind:
remote_file_syncer:
http_client:
max_connections_per_host: 0
max_idle_connections: 40
max_idle_connections_per_host: 2
idle_connection_timeout_seconds: 60
download_url: "https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz"
save_filepath: "/var/tmp/prebid/GeoLite2-City.tar.gz"
tmp_filepath: "/var/tmp/prebid/tmp/GeoLite2-City.tar.gz"
retry_count: 3
retry_interval_ms: 3000
timeout_ms: 300000
update_interval_ms: 0
account_defaults:
events:
enabled: true
Expand Down Expand Up @@ -541,6 +576,8 @@ account_defaults:
cookiedeprecation:
enabled: true
ttl_sec: 86400
geolocation:
enabled: false
tmax_adjustments:
enabled: true
bidder_response_duration_min_ms: 700
Expand Down Expand Up @@ -631,6 +668,7 @@ func TestFullConfig(t *testing.T) {
cmpInts(t, "http_client_cache.idle_connection_timeout_seconds", 3, cfg.CacheClient.IdleConnTimeout)
cmpInts(t, "gdpr.host_vendor_id", 15, cfg.GDPR.HostVendorID)
cmpStrings(t, "gdpr.default_value", "1", cfg.GDPR.DefaultValue)
cmpBools(t, "gdpr.consent_string_means_in_scope", false, cfg.GDPR.ConsentStringMeansInScope)
cmpStrings(t, "host_schain_node.asi", "pbshostcompany.com", cfg.HostSChainNode.ASI)
cmpStrings(t, "host_schain_node.sid", "00001", cfg.HostSChainNode.SID)
cmpStrings(t, "host_schain_node.rid", "BidRequest", cfg.HostSChainNode.RID)
Expand Down Expand Up @@ -890,6 +928,21 @@ func TestFullConfig(t *testing.T) {
cmpStrings(t, "analytics.agma.accounts.0.publisher_id", "publisher-id", cfg.Analytics.Agma.Accounts[0].PublisherId)
cmpStrings(t, "analytics.agma.accounts.0.code", "agma-code", cfg.Analytics.Agma.Accounts[0].Code)
cmpStrings(t, "analytics.agma.accounts.0.site_app_id", "site-or-app-id", cfg.Analytics.Agma.Accounts[0].SiteAppId)

cmpBools(t, "account_defaults.geolocation.enabled", false, cfg.GeoLocation.Enabled)
cmpBools(t, "geolocation.enabled", false, cfg.GeoLocation.Enabled)
cmpStrings(t, "geolocation.type", "maxmind", cfg.GeoLocation.Type)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.http_client.max_connections_per_host", 0, cfg.GeoLocation.Maxmind.RemoteFileSyncer.HttpClient.MaxConnsPerHost)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.http_client.max_idle_connections", 40, cfg.GeoLocation.Maxmind.RemoteFileSyncer.HttpClient.MaxIdleConns)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.http_client.max_idle_connections_per_host", 2, cfg.GeoLocation.Maxmind.RemoteFileSyncer.HttpClient.MaxIdleConnsPerHost)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.http_client.idle_connection_timeout_seconds", 60, cfg.GeoLocation.Maxmind.RemoteFileSyncer.HttpClient.IdleConnTimeout)
cmpStrings(t, "geolocation.maxmind.remote_file_syncer.download_url", "https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz", cfg.GeoLocation.Maxmind.RemoteFileSyncer.DownloadURL)
cmpStrings(t, "geolocation.maxmind.remote_file_syncer.save_filepath", "/var/tmp/prebid/GeoLite2-City.tar.gz", cfg.GeoLocation.Maxmind.RemoteFileSyncer.SaveFilePath)
cmpStrings(t, "geolocation.maxmind.remote_file_syncer.tmp_filepath", "/var/tmp/prebid/tmp/GeoLite2-City.tar.gz", cfg.GeoLocation.Maxmind.RemoteFileSyncer.TmpFilePath)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.retry_count", 3, cfg.GeoLocation.Maxmind.RemoteFileSyncer.RetryCount)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.retry_interval_ms", 3000, cfg.GeoLocation.Maxmind.RemoteFileSyncer.RetryIntervalMillis)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.timeout_ms", 300000, cfg.GeoLocation.Maxmind.RemoteFileSyncer.TimeoutMillis)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.update_interval_ms", 0, cfg.GeoLocation.Maxmind.RemoteFileSyncer.UpdateIntervalMillis)
}

func TestValidateConfig(t *testing.T) {
Expand Down
63 changes: 63 additions & 0 deletions config/countrycode/countrycode.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
package countrycode

import (
"strings"
)

type CountryCode struct {
map2To3 map[string]string
map3To2 map[string]string
}

func New() *CountryCode {
return &CountryCode{
map2To3: make(map[string]string),
map3To2: make(map[string]string),
}
}

// Load loads country code mapping data
func (c *CountryCode) Load(data string) {
toAlpha2 := make(map[string]string)
toAlpha3 := make(map[string]string)
for _, line := range strings.Split(data, "\n") {
if line == "" {
continue
}
fields := strings.Split(line, ",")
if len(fields) < 2 {
continue
}
alpha2 := strings.TrimSpace(fields[0])
alpha3 := strings.TrimSpace(fields[1])
toAlpha2[alpha3] = alpha2
toAlpha3[alpha2] = alpha3
}

c.map2To3 = toAlpha3
c.map3To2 = toAlpha2
}

// ToAlpha3 converts country code alpha2 to alpha3
func (c *CountryCode) ToAlpha3(alpha2 string) string {
return c.map2To3[alpha2]
}

// ToAlpha2 converts country code alpha3 to alpha2
func (c *CountryCode) ToAlpha2(alpha3 string) string {
return c.map3To2[alpha3]
}

var defaultCountryCode = New()

func Load(data string) {
defaultCountryCode.Load(data)
}

func ToAlpha3(alpha2 string) string {
return defaultCountryCode.ToAlpha3(alpha2)
}

func ToAlpha2(alpha3 string) string {
return defaultCountryCode.ToAlpha2(alpha3)
}
60 changes: 60 additions & 0 deletions config/countrycode/countrycode_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package countrycode

import (
"testing"

"github.com/stretchr/testify/assert"
)

func TestCountryCode(t *testing.T) {
Load(`
AD,AND
AE,ARE
AF,AFG
`)
assert.Equal(t, "AND", ToAlpha3("AD"), "map AD to AND")
assert.Equal(t, "AE", ToAlpha2("ARE"), "map ARE to AE")
}

func TestCountryCodeToAlpha3(t *testing.T) {
c := New()
c.Load(`
AD,AND
AE,ARE
AF,AFG
`)
tests := []struct {
input string
expected string
}{
{"AD", "AND"},
{"AE", "ARE"},
{"XX", ""},
{"", ""},
}

for _, test := range tests {
assert.Equal(t, test.expected, c.ToAlpha3(test.input), "map %s to alpha3", test.input)
}
}

func TestCountryCodeToAlpha2(t *testing.T) {
c := New()
c.Load(`
AD,AND
AE,ARE
AF,AFG
`)
tests := []struct {
input string
expected string
}{
{"AND", "AD"},
{"ARE", "AE"},
{"", ""},
}

for _, test := range tests {
assert.Equal(t, test.expected, c.ToAlpha2(test.input), "map %s to alpha2", test.input)
}
}
Loading