Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow adlist duplicates #1886

Merged
merged 2 commits into from
Mar 2, 2024
Merged

Allow adlist duplicates #1886

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7a919cb
Allow narrowing down the (ad)list type using the (optional) query par…
DL6ER Feb 13, 2024
750db7e
Merge branch 'development-v6' into tweak/allow_adlist_dups
DL6ER Feb 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions src/api/docs/content/specs/lists.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ components:
summary: Modify list
parameters:
- $ref: 'lists.yaml#/components/parameters/list'
- $ref: 'lists.yaml#/components/parameters/listtype'
get:
summary: Get lists
tags:
Expand Down Expand Up @@ -449,3 +450,14 @@ components:
required: true
description: Address of the list
example: https://hosts-file.net/ad_servers.txt
listtype:
in: query
name: type
schema:
type: string
enum:
- "allow"
- "block"
required: false
description: Type of list, optional
example: block
61 changes: 52 additions & 9 deletions src/api/list.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ static int api_list_read(struct ftl_conn *api,
sql_msg);
}

tablerow table;
tablerow table = { 0 };
cJSON *rows = JSON_NEW_ARRAY();
while(gravityDB_readTableGetRow(listtype, &table, &sql_msg))
{
Expand All @@ -48,7 +48,9 @@ static int api_list_read(struct ftl_conn *api,
JSON_COPY_STR_TO_OBJECT(row, "name", table.name);
JSON_COPY_STR_TO_OBJECT(row, "comment", table.comment);
}
else if(listtype == GRAVITY_ADLISTS)
else if(listtype == GRAVITY_ADLISTS ||
listtype == GRAVITY_ADLISTS_BLOCK ||
listtype == GRAVITY_ADLISTS_ALLOW)
{
JSON_COPY_STR_TO_OBJECT(row, "address", table.address);
JSON_COPY_STR_TO_OBJECT(row, "comment", table.comment);
Expand Down Expand Up @@ -126,7 +128,9 @@ static int api_list_read(struct ftl_conn *api,
JSON_ADD_NUMBER_TO_OBJECT(row, "date_modified", table.date_modified);

// Properties added in https://github.com/pi-hole/pi-hole/pull/3951
if(listtype == GRAVITY_ADLISTS)
if(listtype == GRAVITY_ADLISTS ||
listtype == GRAVITY_ADLISTS_BLOCK ||
listtype == GRAVITY_ADLISTS_ALLOW)
{
JSON_REF_STR_IN_OBJECT(row, "type", table.type);
JSON_ADD_NUMBER_TO_OBJECT(row, "date_updated", table.date_updated);
Expand All @@ -147,7 +151,9 @@ static int api_list_read(struct ftl_conn *api,
cJSON *json = JSON_NEW_OBJECT();
if(listtype == GRAVITY_GROUPS)
objname = "groups";
else if(listtype == GRAVITY_ADLISTS)
else if(listtype == GRAVITY_ADLISTS ||
listtype == GRAVITY_ADLISTS_BLOCK ||
listtype == GRAVITY_ADLISTS_ALLOW)
objname = "lists";
else if(listtype == GRAVITY_CLIENTS)
objname = "clients";
Expand Down Expand Up @@ -268,6 +274,8 @@ static int api_list_write(struct ftl_conn *api,
}

case GRAVITY_ADLISTS:
case GRAVITY_ADLISTS_BLOCK:
case GRAVITY_ADLISTS_ALLOW:
{
cJSON *json_address = cJSON_GetObjectItemCaseSensitive(api->payload.json, "address");
if(cJSON_IsString(json_address) && strlen(json_address->valuestring) > 0)
Expand Down Expand Up @@ -331,6 +339,10 @@ static int api_list_write(struct ftl_conn *api,
NULL);
}
}
else if(listtype == GRAVITY_ADLISTS_BLOCK)
row.type_int = ADLIST_BLOCK;
else if(listtype == GRAVITY_ADLISTS_ALLOW)
row.type_int = ADLIST_ALLOW;
else
{
cJSON *json_type = cJSON_GetObjectItemCaseSensitive(api->payload.json, "type");
Expand Down Expand Up @@ -552,7 +564,9 @@ static int api_list_remove(struct ftl_conn *api,
if(listtype == GRAVITY_DOMAINLIST_ALLOW_EXACT ||
listtype == GRAVITY_DOMAINLIST_DENY_EXACT ||
listtype == GRAVITY_DOMAINLIST_ALLOW_REGEX ||
listtype == GRAVITY_DOMAINLIST_DENY_REGEX)
listtype == GRAVITY_DOMAINLIST_DENY_REGEX ||
listtype == GRAVITY_ADLISTS_BLOCK ||
listtype == GRAVITY_ADLISTS_ALLOW)
{
int type = -1;
switch (listtype)
Expand All @@ -568,12 +582,17 @@ static int api_list_remove(struct ftl_conn *api,
break;
case GRAVITY_DOMAINLIST_DENY_REGEX:
type = 3;
break;
case GRAVITY_ADLISTS_BLOCK:
type = ADLIST_BLOCK;
break;
case GRAVITY_ADLISTS_ALLOW:
type = ADLIST_ALLOW;
break;
// Not handled herein
case GRAVITY_GROUPS:
case GRAVITY_ADLISTS:
case GRAVITY_CLIENTS:
// No type required for these tables
break;
// Aggregate types cannot be handled by this routine
case GRAVITY_GRAVITY:
case GRAVITY_ANTIGRAVITY:
case GRAVITY_DOMAINLIST_ALLOW_ALL:
Expand All @@ -582,7 +601,7 @@ static int api_list_remove(struct ftl_conn *api,
case GRAVITY_DOMAINLIST_ALL_REGEX:
case GRAVITY_DOMAINLIST_ALL_ALL:
default:
return false;
break;
}

// Create new JSON array with the item and type:
Expand Down Expand Up @@ -827,6 +846,30 @@ int api_list(struct ftl_conn *api)
api->request->local_uri_raw);
}

// If this is a request for a list, we check if there is a request
// parameter narrowing down which kind of list. If so, we modify the
// list type accordingly
if(listtype == GRAVITY_ADLISTS && api->request->query_string != NULL)
{
// Check if there is a type parameter
char typestr[16] = { 0 };
if(get_string_var(api->request->query_string, "type", typestr, sizeof(typestr)) > 0)
{
if(strcasecmp(typestr, "allow") == 0)
listtype = GRAVITY_ADLISTS_ALLOW;
else if(strcasecmp(typestr, "block") == 0)
listtype = GRAVITY_ADLISTS_BLOCK;
else
{
// Invalid type parameter
return send_json_error(api, 400,
"bad_request",
"Invalid request: Invalid type parameter (should be either \"allow\" or \"block\")",
api->request->query_string);
}
}
}

if(api->method == HTTP_GET)
{
// Read list item identified by URI (or read them all)
Expand Down
101 changes: 75 additions & 26 deletions src/database/gravity-db.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1575,6 +1575,8 @@ bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row,
// Nothing to be done for these tables
case GRAVITY_GROUPS:
case GRAVITY_ADLISTS:
case GRAVITY_ADLISTS_BLOCK:
case GRAVITY_ADLISTS_ALLOW:
case GRAVITY_CLIENTS:
break;

Expand All @@ -1597,15 +1599,17 @@ bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row,
{
querystr = "INSERT INTO \"group\" (name,enabled,description) VALUES (:item,:enabled,:comment);";
}
else if(listtype == GRAVITY_ADLISTS)
else if(listtype == GRAVITY_ADLISTS ||
listtype == GRAVITY_ADLISTS_BLOCK ||
listtype == GRAVITY_ADLISTS_ALLOW)
{
querystr = "INSERT INTO adlist (address,enabled,comment,type) VALUES (:item,:enabled,:comment,:type);";
}
else if(listtype == GRAVITY_CLIENTS)
{
querystr = "INSERT INTO client (ip,comment) VALUES (:item,:comment);";
}
else // domainlis
else // domainlist
{
querystr = "INSERT INTO domainlist (domain,type,enabled,comment) VALUES (:item,:type,:enabled,:comment);";
}
Expand All @@ -1625,9 +1629,11 @@ bool gravityDB_addToTable(const enum gravity_list_type listtype, tablerow *row,
querystr = "UPDATE \"group\" SET name = :name, enabled = :enabled, description = :comment "
"WHERE name = :item";
}
else if(listtype == GRAVITY_ADLISTS)
else if(listtype == GRAVITY_ADLISTS ||
listtype == GRAVITY_ADLISTS_BLOCK ||
listtype == GRAVITY_ADLISTS_ALLOW)
querystr = "INSERT INTO adlist (address,enabled,comment,type) VALUES (:item,:enabled,:comment,:type) "\
"ON CONFLICT(address) DO UPDATE SET enabled = :enabled, comment = :comment, type = :type;";
"ON CONFLICT(address,type) DO UPDATE SET enabled = :enabled, comment = :comment, type = :type;";
else if(listtype == GRAVITY_CLIENTS)
querystr = "INSERT INTO client (ip,comment) VALUES (:item,:comment) "\
"ON CONFLICT(ip) DO UPDATE SET comment = :comment;";
Expand Down Expand Up @@ -1825,11 +1831,14 @@ bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON*
return false;
}

const bool isDomain = listtype == GRAVITY_DOMAINLIST_ALLOW_EXACT ||
listtype == GRAVITY_DOMAINLIST_DENY_EXACT ||
listtype == GRAVITY_DOMAINLIST_ALLOW_REGEX ||
listtype == GRAVITY_DOMAINLIST_DENY_REGEX ||
listtype == GRAVITY_DOMAINLIST_ALL_ALL; // batch delete
const bool hasType = listtype == GRAVITY_DOMAINLIST_ALLOW_EXACT ||
listtype == GRAVITY_DOMAINLIST_DENY_EXACT ||
listtype == GRAVITY_DOMAINLIST_ALLOW_REGEX ||
listtype == GRAVITY_DOMAINLIST_DENY_REGEX ||
listtype == GRAVITY_DOMAINLIST_ALL_ALL ||
listtype == GRAVITY_ADLISTS ||
listtype == GRAVITY_ADLISTS_BLOCK ||
listtype == GRAVITY_ADLISTS_ALLOW;

// Begin transaction
const char *querystr = "BEGIN TRANSACTION;";
Expand All @@ -1843,7 +1852,7 @@ bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON*
}

// Create temporary table for JSON argument
if(isDomain)
if(hasType)
// Create temporary table for domains to be deleted
querystr = "CREATE TEMPORARY TABLE deltable (type INT, item TEXT);";
else
Expand Down Expand Up @@ -1885,7 +1894,7 @@ bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON*
sqlite3_finalize(stmt);

// Prepare statement for inserting items into virtual table
if(isDomain)
if(hasType)
querystr = "INSERT INTO deltable (type, item) VALUES (:type, :item);";
else
querystr = "INSERT INTO deltable (item) VALUES (:item);";
Expand All @@ -1910,12 +1919,24 @@ bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON*
{
// Bind type to prepared statement
cJSON *type = cJSON_GetObjectItemCaseSensitive(it, "type");
int type_int = cJSON_IsNumber(type) ? type->valueint : -1;
if(listtype == GRAVITY_ADLISTS_BLOCK)
type_int = ADLIST_BLOCK;
else if(listtype == GRAVITY_ADLISTS_ALLOW)
type_int = ADLIST_ALLOW;
else if(listtype == GRAVITY_ADLISTS && cJSON_IsString(type))
{
if(strcasecmp(type->valuestring, "block") == 0)
type_int = ADLIST_BLOCK;
else if(strcasecmp(type->valuestring, "allow") == 0)
type_int = ADLIST_ALLOW;
}
const int type_idx = sqlite3_bind_parameter_index(stmt, ":type");
if(type_idx > 0 && (!cJSON_IsNumber(type) || (rc = sqlite3_bind_int(stmt, type_idx, type->valueint)) != SQLITE_OK))
if(type_idx > 0 && (rc = sqlite3_bind_int(stmt, type_idx, type_int)) != SQLITE_OK)
{
*message = sqlite3_errmsg(gravity_db);
log_err("gravityDB_delFromTable(%d): Failed to bind type (error %d) - %s",
type->valueint, rc, *message);
type_int, rc, *message);
sqlite3_reset(stmt);
sqlite3_finalize(stmt);

Expand Down Expand Up @@ -1981,12 +2002,15 @@ bool gravityDB_delFromTable(const enum gravity_list_type listtype, const cJSON*
const char *querystrs[4] = {NULL, NULL, NULL, NULL};
if(listtype == GRAVITY_GROUPS)
querystrs[0] = "DELETE FROM \"group\" WHERE name IN (SELECT item FROM deltable);";
else if(listtype == GRAVITY_ADLISTS)
else if(listtype == GRAVITY_ADLISTS ||
listtype == GRAVITY_ADLISTS_BLOCK ||
listtype == GRAVITY_ADLISTS_ALLOW)
{
// This is actually a three-step deletion to satisfy foreign-key constraints
querystrs[0] = "DELETE FROM gravity WHERE adlist_id IN (SELECT id FROM adlist WHERE address IN (SELECT item FROM deltable));";
querystrs[1] = "DELETE FROM antigravity WHERE adlist_id IN (SELECT id FROM adlist WHERE address IN (SELECT item FROM deltable));";
querystrs[2] = "DELETE FROM adlist WHERE address IN (SELECT item FROM deltable);";
// This is actually a four-step deletion to satisfy foreign-key constraints
querystrs[0] = "DELETE FROM gravity WHERE adlist_id IN (SELECT id FROM adlist WHERE address IN (SELECT item FROM deltable WHERE type = 0));";
querystrs[1] = "DELETE FROM antigravity WHERE adlist_id IN (SELECT id FROM adlist WHERE address IN (SELECT item FROM deltable WHERE type = 1));";
querystrs[2] = "DELETE FROM adlist WHERE address IN (SELECT item FROM deltable WHERE type = 0) AND type = 0;";
querystrs[3] = "DELETE FROM adlist WHERE address IN (SELECT item FROM deltable WHERE type = 1) AND type = 1;";
}
else if(listtype == GRAVITY_CLIENTS)
querystrs[0] = "DELETE FROM client WHERE ip IN (SELECT item FROM deltable);";
Expand Down Expand Up @@ -2096,12 +2120,16 @@ bool gravityDB_readTable(const enum gravity_list_type listtype,
case GRAVITY_DOMAINLIST_ALL_ALL:
type = "0,1,2,3";
break;

// No type required for these tables
case GRAVITY_GRAVITY:
case GRAVITY_ANTIGRAVITY:
case GRAVITY_GROUPS:
case GRAVITY_ADLISTS:
case GRAVITY_CLIENTS:
// No type required for these tables
case GRAVITY_ADLISTS:
// Type is set in the SQL query directly
case GRAVITY_ADLISTS_BLOCK:
case GRAVITY_ADLISTS_ALLOW:
break;
}

Expand Down Expand Up @@ -2133,19 +2161,29 @@ bool gravityDB_readTable(const enum gravity_list_type listtype,
}
snprintf(querystr, buflen, "SELECT id,name,enabled,date_added,date_modified,description AS comment FROM \"group\"%s;", filter);
}
else if(listtype == GRAVITY_ADLISTS)
else if(listtype == GRAVITY_ADLISTS ||
listtype == GRAVITY_ADLISTS_BLOCK ||
listtype == GRAVITY_ADLISTS_ALLOW)
{
if(listtype == GRAVITY_ADLISTS_BLOCK)
filter = "type = 0";
else if(listtype == GRAVITY_ADLISTS_ALLOW)
filter = "type = 1";
else
filter = "TRUE";

const char *filter2 = "";
if(item != NULL && item[0] != '\0')
{
if(exact)
filter = " WHERE address = :item";
filter2 = " AND address = :item";
else
filter = " WHERE address LIKE :item";
filter2 = " AND address LIKE :item";
}
snprintf(querystr, buflen, "SELECT id,type,address,enabled,date_added,date_modified,comment,"
"(SELECT GROUP_CONCAT(group_id) FROM adlist_by_group g WHERE g.adlist_id = a.id) AS group_ids,"
"date_updated,number,invalid_domains,status,abp_entries "
"FROM adlist a%s;", filter);
"FROM adlist a WHERE %s%s;", filter, filter2);
}
else if(listtype == GRAVITY_CLIENTS)
{
Expand Down Expand Up @@ -2312,6 +2350,8 @@ bool gravityDB_readTableGetRow(const enum gravity_list_type listtype, tablerow *
}
}
else if(listtype == GRAVITY_ADLISTS ||
listtype == GRAVITY_ADLISTS_ALLOW ||
listtype == GRAVITY_ADLISTS_BLOCK ||
listtype == GRAVITY_GRAVITY ||
listtype == GRAVITY_ANTIGRAVITY)
{
Expand All @@ -2328,6 +2368,10 @@ bool gravityDB_readTableGetRow(const enum gravity_list_type listtype, tablerow *
break;
}
}
else
{
row->type = "unknown";
}
}

else if(strcasecmp(cname, "domain") == 0)
Expand Down Expand Up @@ -2425,9 +2469,14 @@ bool gravityDB_edit_groups(const enum gravity_list_type listtype, cJSON *groups,
del_querystr = "DELETE FROM client_by_group WHERE client_id = :id;";
add_querystr = "INSERT INTO client_by_group (client_id,group_id) VALUES (:id,:gid);";
}
else if(listtype == GRAVITY_ADLISTS)
else if(listtype == GRAVITY_ADLISTS ||
listtype == GRAVITY_ADLISTS_BLOCK ||
listtype == GRAVITY_ADLISTS_ALLOW)
{
get_querystr = "SELECT id FROM adlist WHERE address = :item";
if(listtype == GRAVITY_ADLISTS)
get_querystr = "SELECT id FROM adlist WHERE address = :item";
else
get_querystr = "SELECT id FROM adlist WHERE address = :item AND type = :type";
del_querystr = "DELETE FROM adlist_by_group WHERE adlist_id = :id;";
add_querystr = "INSERT INTO adlist_by_group (adlist_id,group_id) VALUES (:id,:gid);";
}
Expand Down
4 changes: 3 additions & 1 deletion src/enums.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,9 @@ enum gravity_list_type {
GRAVITY_ADLISTS,
GRAVITY_CLIENTS,
GRAVITY_GRAVITY,
GRAVITY_ANTIGRAVITY
GRAVITY_ANTIGRAVITY,
GRAVITY_ADLISTS_BLOCK,
GRAVITY_ADLISTS_ALLOW
} __attribute__ ((packed));

enum gravity_tables {
Expand Down
Loading
Loading