bol-van/zapret
1
1
Fork 0
mirror of https://github.com/bol-van/zapret.git synced 2024-12-18 22:11:31 +03:00
Code Issues Actions Packages Projects Releases Wiki Activity

tpws: hostlist/ipset dedup and auto reload

Browse Source
This commit is contained in:
bol-van 2024-10-29 15:08:05 +03:00
parent 1fdf5477b4
commit 2464d27550
11 changed files with 577 additions and 228 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

195
tpws/hostlist.c
View File

@ -1,7 +1,6 @@
#include <stdio.h>
#include "hostlist.h"
#include "gzip.h"
#include "params.h"
#include "helpers.h"
// inplace tolower() and add to pool
@ -56,7 +55,7 @@ bool AppendHostList(strpool **hostlist, const char *filename)
if (r==Z_OK)
{
DLOG_CONDUP("zlib compression detected. uncompressed size : %zu\n", zsize);
p = zbuf;
e = zbuf + zsize;
while(p<e)
@ -79,7 +78,7 @@ bool AppendHostList(strpool **hostlist, const char *filename)
else
{
DLOG_CONDUP("loading plain text list\n");
while (fgets(s, sizeof(s), F))
{
p = s;
@ -97,22 +96,38 @@ bool AppendHostList(strpool **hostlist, const char *filename)
return true;
}
bool LoadHostLists(strpool **hostlist, struct str_list_head *file_list)
static bool LoadHostList(struct hostlist_file *hfile)
{
struct str_list *file;
if (*hostlist)
time_t t = file_mod_time(hfile->filename);
if (!t)
{
StrPoolDestroy(hostlist);
*hostlist = NULL;
// stat() error
DLOG_ERR("cannot access hostlist file '%s'. in-memory content remains unchanged.\n",hfile->filename);
return true;
}
LIST_FOREACH(file, file_list, next)
if (t==hfile->mod_time) return true; // up to date
StrPoolDestroy(&hfile->hostlist);
if (!AppendHostList(&hfile->hostlist, hfile->filename))
{
if (!AppendHostList(hostlist, file->str)) return false;
StrPoolDestroy(&hfile->hostlist);
return false;
}
hfile->mod_time=t;
return true;
}
static bool LoadHostLists(struct hostlist_files_head *list)
{
bool bres=true;
struct hostlist_file *hfile;
LIST_FOREACH(hfile, list, next)
{
if (!LoadHostList(hfile))
// at least one failed
bres=false;
}
return bres;
}
bool NonEmptyHostlist(strpool **hostlist)
{
@ -120,8 +135,27 @@ bool NonEmptyHostlist(strpool **hostlist)
return *hostlist ? true : StrPoolAddStrLen(hostlist, "@&()", 4);
}
static void MakeAutolistsNonEmpty()
{
struct desync_profile_list *dpl;
LIST_FOREACH(dpl, &params.desync_profiles, next)
{
if (dpl->dp.hostlist_auto)
NonEmptyHostlist(&dpl->dp.hostlist_auto->hostlist);
}
}
bool SearchHostList(strpool *hostlist, const char *host)
bool LoadAllHostLists()
{
if (!LoadHostLists(&params.hostlists))
return false;
MakeAutolistsNonEmpty();
return true;
}
static bool SearchHostList(strpool *hostlist, const char *host)
{
if (hostlist)
{
@ -130,7 +164,7 @@ bool SearchHostList(strpool *hostlist, const char *host)
while (p)
{
bInHostList = StrPoolCheckStr(hostlist, p);
VPRINT("Hostlist check for %s : %s\n", p, bInHostList ? "positive" : "negative");
VPRINT("hostlist check for %s : %s\n", p, bInHostList ? "positive" : "negative");
if (bInHostList) return true;
p = strchr(p, '.');
if (p) p++;
@ -139,73 +173,108 @@ bool SearchHostList(strpool *hostlist, const char *host)
return false;
}
// return : true = apply fooling, false = do not apply
static bool HostlistCheck_(strpool *hostlist, strpool *hostlist_exclude, const char *host, bool *excluded)
static bool HostlistsReloadCheck(const struct hostlist_collection_head *hostlists)
{
if (excluded) *excluded = false;
if (hostlist_exclude)
struct hostlist_item *item;
LIST_FOREACH(item, hostlists, next)
{
VPRINT("Checking exclude hostlist\n");
if (SearchHostList(hostlist_exclude, host))
if (!LoadHostList(item->hfile))
return false;
}
MakeAutolistsNonEmpty();
return true;
}
bool HostlistsReloadCheckForProfile(const struct desync_profile *dp)
{
return HostlistsReloadCheck(&dp->hl_collection) && HostlistsReloadCheck(&dp->hl_collection_exclude);
}
// return : true = apply fooling, false = do not apply
static bool HostlistCheck_(const struct hostlist_collection_head *hostlists, const struct hostlist_collection_head *hostlists_exclude, const char *host, bool *excluded, bool bSkipReloadCheck)
{
struct hostlist_item *item;
if (excluded) *excluded = false;
if (!bSkipReloadCheck)
if (!HostlistsReloadCheck(hostlists) || !HostlistsReloadCheck(hostlists_exclude))
return false;
LIST_FOREACH(item, hostlists_exclude, next)
{
VPRINT("[%s] exclude ", item->hfile->filename);
if (SearchHostList(item->hfile->hostlist, host))
{
if (excluded) *excluded = true;
return false;
}
}
if (hostlist)
// old behavior compat: all include lists are empty means check passes
if (!hostlist_collection_is_empty(hostlists))
{
VPRINT("Checking include hostlist\n");
return SearchHostList(hostlist, host);
}
return true;
}
static bool LoadIncludeHostListsForProfile(struct desync_profile *dp)
{
if (!LoadHostLists(&dp->hostlist, &dp->hostlist_files))
LIST_FOREACH(item, hostlists, next)
{
VPRINT("[%s] include ", item->hfile->filename);
if (SearchHostList(item->hfile->hostlist, host))
return true;
}
return false;
if (*dp->hostlist_auto_filename)
{
dp->hostlist_auto_mod_time = file_mod_time(dp->hostlist_auto_filename);
NonEmptyHostlist(&dp->hostlist);
}
return true;
}
bool HostlistCheck(struct desync_profile *dp, const char *host, bool *excluded)
// return : true = apply fooling, false = do not apply
bool HostlistCheck(const struct desync_profile *dp, const char *host, bool *excluded, bool bSkipReloadCheck)
{
VPRINT("* hostlist check for profile %d\n",dp->n);
if (*dp->hostlist_auto_filename)
{
time_t t = file_mod_time(dp->hostlist_auto_filename);
if (t!=dp->hostlist_auto_mod_time)
{
DLOG_CONDUP("Autohostlist '%s' from profile %d was modified. Reloading include hostlists for this profile.\n",dp->hostlist_auto_filename, dp->n);
if (!LoadIncludeHostListsForProfile(dp))
{
// what will we do without hostlist ?? sure, gonna die
exit(1);
}
dp->hostlist_auto_mod_time = t;
NonEmptyHostlist(&dp->hostlist);
}
}
return HostlistCheck_(dp->hostlist, dp->hostlist_exclude, host, excluded);
return HostlistCheck_(&dp->hl_collection, &dp->hl_collection_exclude, host, excluded, bSkipReloadCheck);
}
bool LoadIncludeHostLists()
static struct hostlist_file *RegisterHostlist_(struct hostlist_files_head *hostlists, struct hostlist_collection_head *hl_collection, const char *filename)
{
struct desync_profile_list *dpl;
LIST_FOREACH(dpl, &params.desync_profiles, next)
if (!LoadIncludeHostListsForProfile(&dpl->dp))
return false;
return true;
struct hostlist_file *hfile;
if (!(hfile=hostlist_files_search(hostlists, filename)))
if (!(hfile=hostlist_files_add(hostlists, filename)))
return NULL;
if (!hostlist_collection_search(hl_collection, filename))
if (!hostlist_collection_add(hl_collection, hfile))
return NULL;
return hfile;
}
bool LoadExcludeHostLists()
struct hostlist_file *RegisterHostlist(struct desync_profile *dp, bool bExclude, const char *filename)
{
struct desync_profile_list *dpl;
LIST_FOREACH(dpl, &params.desync_profiles, next)
if (!LoadHostLists(&dpl->dp.hostlist_exclude, &dpl->dp.hostlist_exclude_files))
return false;
return true;
if (!file_mod_time(filename))
{
DLOG_ERR("cannot access hostlist file '%s'\n",filename);
return NULL;
}
return RegisterHostlist_(
&params.hostlists,
bExclude ? &dp->hl_collection_exclude : &dp->hl_collection,
filename);
}
void HostlistsDebug()
{
if (!params.debug) return;
struct hostlist_file *hfile;
struct desync_profile_list *dpl;
struct hostlist_item *hl_item;
LIST_FOREACH(hfile, &params.hostlists, next)
VPRINT("hostlist file %s%s\n",hfile->filename,hfile->hostlist ? "" : " (empty)");
LIST_FOREACH(dpl, &params.desync_profiles, next)
{
LIST_FOREACH(hl_item, &dpl->dp.hl_collection, next)
if (hl_item->hfile!=dpl->dp.hostlist_auto)
VPRINT("profile %d include hostlist %s%s\n",dpl->dp.n, hl_item->hfile->filename,hl_item->hfile->hostlist ? "" : " (empty)");
LIST_FOREACH(hl_item, &dpl->dp.hl_collection_exclude, next)
VPRINT("profile %d exclude hostlist %s%s\n",dpl->dp.n,hl_item->hfile->filename,hl_item->hfile->hostlist ? "" : " (empty)");
if (dpl->dp.hostlist_auto)
VPRINT("profile %d auto hostlist %s%s\n",dpl->dp.n,dpl->dp.hostlist_auto->filename,dpl->dp.hostlist_auto->hostlist ? "" : " (empty)");
}
}

10
tpws/hostlist.h
View File

@ -5,10 +5,10 @@
#include "params.h"
bool AppendHostList(strpool **hostlist, const char *filename);
bool LoadHostLists(strpool **hostlist, struct str_list_head *file_list);
bool LoadIncludeHostLists();
bool LoadExcludeHostLists();
bool LoadAllHostLists();
bool NonEmptyHostlist(strpool **hostlist);
bool SearchHostList(strpool *hostlist, const char *host);
// return : true = apply fooling, false = do not apply
bool HostlistCheck(struct desync_profile *dp,const char *host, bool *excluded);
bool HostlistCheck(const struct desync_profile *dp,const char *host, bool *excluded, bool bSkipReloadCheck);
struct hostlist_file *RegisterHostlist(struct desync_profile *dp, bool bExclude, const char *filename);
bool HostlistsReloadCheckForProfile(const struct desync_profile *dp);
void HostlistsDebug();

162
tpws/ipset.c
View File

@ -3,6 +3,7 @@
#include "gzip.h"
#include "helpers.h"
// inplace tolower() and add to pool
static bool addpool(ipset *ips, char **s, const char *end, int *ct)
{
@ -75,7 +76,7 @@ static bool AppendIpset(ipset *ips, const char *filename)
if (r==Z_OK)
{
DLOG_CONDUP("zlib compression detected. uncompressed size : %zu\n", zsize);
p = zbuf;
e = zbuf + zsize;
while(p<e)
@ -98,7 +99,7 @@ static bool AppendIpset(ipset *ips, const char *filename)
else
{
DLOG_CONDUP("loading plain text list\n");
while (fgets(s, sizeof(s)-1, F))
{
p = s;
@ -116,37 +117,45 @@ static bool AppendIpset(ipset *ips, const char *filename)
return true;
}
static bool LoadIpsets(ipset *ips, struct str_list_head *file_list)
static bool LoadIpset(struct ipset_file *hfile)
{
struct str_list *file;
ipsetDestroy(ips);
LIST_FOREACH(file, file_list, next)
time_t t = file_mod_time(hfile->filename);
if (!t)
{
if (!AppendIpset(ips, file->str)) return false;
// stat() error
DLOG_ERR("cannot access ipset file '%s'. in-memory content remains unchanged.\n",hfile->filename);
return true;
}
if (t==hfile->mod_time) return true; // up to date
ipsetDestroy(&hfile->ipset);
if (!AppendIpset(&hfile->ipset, hfile->filename))
{
ipsetDestroy(&hfile->ipset);
return false;
}
hfile->mod_time=t;
return true;
}
static bool LoadIpsets(struct ipset_files_head *list)
{
bool bres=true;
struct ipset_file *hfile;
LIST_FOREACH(hfile, list, next)
{
if (!LoadIpset(hfile))
// at least one failed
bres=false;
}
return bres;
}
bool LoadIncludeIpsets()
bool LoadAllIpsets()
{
struct desync_profile_list *dpl;
LIST_FOREACH(dpl, &params.desync_profiles, next)
if (!LoadIpsets(&dpl->dp.ips, &dpl->dp.ipset_files))
return false;
return true;
}
bool LoadExcludeIpsets()
{
struct desync_profile_list *dpl;
LIST_FOREACH(dpl, &params.desync_profiles, next)
if (!LoadIpsets(&dpl->dp.ips_exclude, &dpl->dp.ipset_exclude_files))
return false;
return true;
return LoadIpsets(&params.ipsets);
}
bool SearchIpset(const ipset *ips, const struct in_addr *ipv4, const struct in6_addr *ipv6)
static bool SearchIpset(const ipset *ips, const struct in_addr *ipv4, const struct in6_addr *ipv6)
{
char s_ip[40];
bool bInSet=false;
@ -172,24 +181,109 @@ bool SearchIpset(const ipset *ips, const struct in_addr *ipv4, const struct in6_
return bInSet;
}
static bool IpsetCheck_(const ipset *ips, const ipset *ips_exclude, const struct in_addr *ipv4, const struct in6_addr *ipv6)
static bool IpsetsReloadCheck(const struct ipset_collection_head *ipsets)
{
if (!IPSET_EMPTY(ips_exclude))
struct ipset_item *item;
LIST_FOREACH(item, ipsets, next)
{
VPRINT("exclude ");
if (SearchIpset(ips_exclude, ipv4, ipv6))
if (!LoadIpset(item->hfile))
return false;
}
if (!IPSET_EMPTY(ips))
return true;
}
bool IpsetsReloadCheckForProfile(const struct desync_profile *dp)
{
return IpsetsReloadCheck(&dp->ips_collection) && IpsetsReloadCheck(&dp->ips_collection_exclude);
}
static bool IpsetCheck_(const struct ipset_collection_head *ips, const struct ipset_collection_head *ips_exclude, const struct in_addr *ipv4, const struct in6_addr *ipv6)
{
struct ipset_item *item;
if (!IpsetsReloadCheck(ips) || !IpsetsReloadCheck(ips_exclude))
return false;
LIST_FOREACH(item, ips_exclude, next)
{
VPRINT("include ");
return SearchIpset(ips, ipv4, ipv6);
VPRINT("[%s] exclude ",item->hfile->filename);
if (SearchIpset(&item->hfile->ipset, ipv4, ipv6))
return false;
}
// old behavior compat: all include lists are empty means check passes
if (!ipset_collection_is_empty(ips))
{
LIST_FOREACH(item, ips, next)
{
VPRINT("[%s] include ",item->hfile->filename);
if (SearchIpset(&item->hfile->ipset, ipv4, ipv6))
return true;
}
return false;
}
return true;
}
bool IpsetCheck(struct desync_profile *dp, const struct in_addr *ipv4, const struct in6_addr *ipv6)
bool IpsetCheck(const struct desync_profile *dp, const struct in_addr *ipv4, const struct in6_addr *ipv6)
{
if (!PROFILE_IPSETS_EMPTY(dp)) VPRINT("* ipset check for profile %d\n",dp->n);
return IpsetCheck_(&dp->ips,&dp->ips_exclude,ipv4,ipv6);
if (PROFILE_IPSETS_EMPTY(dp)) return true;
VPRINT("* ipset check for profile %d\n",dp->n);
return IpsetCheck_(&dp->ips_collection,&dp->ips_collection_exclude,ipv4,ipv6);
}
static struct ipset_file *RegisterIpset_(struct ipset_files_head *ipsets, struct ipset_collection_head *ips_collection, const char *filename)
{
struct ipset_file *hfile;
if (!(hfile=ipset_files_search(ipsets, filename)))
if (!(hfile=ipset_files_add(ipsets, filename)))
return NULL;
if (!ipset_collection_search(ips_collection, filename))
if (!ipset_collection_add(ips_collection, hfile))
return NULL;
return hfile;
}
struct ipset_file *RegisterIpset(struct desync_profile *dp, bool bExclude, const char *filename)
{
if (!file_mod_time(filename))
{
DLOG_ERR("cannot access ipset file '%s'\n",filename);
return NULL;
}
return RegisterIpset_(
&params.ipsets,
bExclude ? &dp->ips_collection_exclude : &dp->ips_collection,
filename);
}
static const char *dbg_ipset_fill(const ipset *ips)
{
if (ips->ips4)
if (ips->ips6)
return "ipv4+ipv6";
else
return "ipv4";
else
if (ips->ips6)
return "ipv6";
else
return "empty";
}
void IpsetsDebug()
{
if (!params.debug) return;
struct ipset_file *hfile;
struct desync_profile_list *dpl;
struct ipset_item *ips_item;
LIST_FOREACH(hfile, &params.ipsets, next)
VPRINT("ipset file %s (%s)\n",hfile->filename,dbg_ipset_fill(&hfile->ipset));
LIST_FOREACH(dpl, &params.desync_profiles, next)
{
LIST_FOREACH(ips_item, &dpl->dp.ips_collection, next)
VPRINT("profile %d include ipset %s (%s)\n",dpl->dp.n,ips_item->hfile->filename,dbg_ipset_fill(&ips_item->hfile->ipset));
LIST_FOREACH(ips_item, &dpl->dp.ips_collection_exclude, next)
VPRINT("profile %d exclude ipset %s (%s)\n",dpl->dp.n,ips_item->hfile->filename,dbg_ipset_fill(&ips_item->hfile->ipset));
}
}

8
tpws/ipset.h
View File

@ -5,7 +5,7 @@
#include "params.h"
#include "pools.h"
bool LoadIncludeIpsets();
bool LoadExcludeIpsets();
bool SearchIpset(const ipset *ips, const struct in_addr *ipv4, const struct in6_addr *ipv6);
bool IpsetCheck(struct desync_profile *dp, const struct in_addr *ipv4, const struct in6_addr *ipv6);
bool LoadAllIpsets();
bool IpsetCheck(const struct desync_profile *dp, const struct in_addr *ipv4, const struct in6_addr *ipv6);
struct ipset_file *RegisterIpset(struct desync_profile *dp, bool bExclude, const char *filename);
void IpsetsDebug();

28
tpws/params.c
View File

@ -145,10 +145,12 @@ struct desync_profile_list *dp_list_add(struct desync_profile_list_head *head)
struct desync_profile_list *entry = calloc(1,sizeof(struct desync_profile_list));
if (!entry) return NULL;
LIST_INIT(&entry->dp.hostlist_files);
LIST_INIT(&entry->dp.hostlist_exclude_files);
entry->dp.filter_ipv4 = entry->dp.filter_ipv6 = true;
LIST_INIT(&entry->dp.hl_collection);
LIST_INIT(&entry->dp.hl_collection_exclude);
LIST_INIT(&entry->dp.ips_collection);
LIST_INIT(&entry->dp.ips_collection_exclude);
entry->dp.filter_ipv4 = entry->dp.filter_ipv6 = true;
memcpy(entry->dp.hostspell, "host", 4); // default hostspell
entry->dp.hostlist_auto_fail_threshold = HOSTLIST_AUTO_FAIL_THRESHOLD_DEFAULT;
entry->dp.hostlist_auto_fail_time = HOSTLIST_AUTO_FAIL_TIME_DEFAULT;
@ -167,14 +169,10 @@ struct desync_profile_list *dp_list_add(struct desync_profile_list_head *head)
}
static void dp_entry_destroy(struct desync_profile_list *entry)
{
strlist_destroy(&entry->dp.hostlist_files);
strlist_destroy(&entry->dp.hostlist_exclude_files);
strlist_destroy(&entry->dp.ipset_files);
strlist_destroy(&entry->dp.ipset_exclude_files);
StrPoolDestroy(&entry->dp.hostlist_exclude);
StrPoolDestroy(&entry->dp.hostlist);
ipsetDestroy(&entry->dp.ips);
ipsetDestroy(&entry->dp.ips_exclude);
hostlist_collection_destroy(&entry->dp.hl_collection);
hostlist_collection_destroy(&entry->dp.hl_collection_exclude);
ipset_collection_destroy(&entry->dp.ips_collection);
ipset_collection_destroy(&entry->dp.ips_collection_exclude);
HostFailPoolDestroy(&entry->dp.hostlist_auto_fail_counters);
free(entry);
}
@ -187,11 +185,3 @@ void dp_list_destroy(struct desync_profile_list_head *head)
dp_entry_destroy(entry);
}
}
bool dp_list_have_autohostlist(struct desync_profile_list_head *head)
{
struct desync_profile_list *dpl;
LIST_FOREACH(dpl, head, next)
if (*dpl->dp.hostlist_auto_filename)
return true;
return false;
}

27
tpws/params.h
View File

@ -48,22 +48,25 @@ struct desync_profile
bool tamper_start_n,tamper_cutoff_n;
unsigned int tamper_start,tamper_cutoff;
bool filter_ipv4,filter_ipv6;
port_filter pf_tcp;
uint32_t filter_l7; // L7_PROTO_* bits
ipset ips,ips_exclude;
struct str_list_head ipset_files, ipset_exclude_files;
strpool *hostlist, *hostlist_exclude;
struct str_list_head hostlist_files, hostlist_exclude_files;
char hostlist_auto_filename[PATH_MAX];
int hostlist_auto_fail_threshold, hostlist_auto_fail_time;
time_t hostlist_auto_mod_time;
// list of pointers to ipsets
struct ipset_collection_head ips_collection, ips_collection_exclude;
// list of pointers to hostlist files
struct hostlist_collection_head hl_collection, hl_collection_exclude;
// pointer to autohostlist. NULL if no autohostlist for the profile.
struct hostlist_file *hostlist_auto;
int hostlist_auto_fail_threshold, hostlist_auto_fail_time, hostlist_auto_retrans_threshold;
hostfail_pool *hostlist_auto_fail_counters;
};
#define PROFILE_IPSETS_EMPTY(dp) (IPSET_EMPTY(&dp->ips) && IPSET_EMPTY(&dp->ips_exclude))
#define PROFILE_IPSETS_EMPTY(dp) (ipset_collection_is_empty(&dp->ips_collection) && ipset_collection_is_empty(&dp->ips_collection_exclude))
#define PROFILE_HOSTLISTS_EMPTY(dp) (hostlist_collection_is_empty(&dp->hl_collection) && hostlist_collection_is_empty(&dp->hl_collection_exclude))
struct desync_profile_list {
struct desync_profile dp;
@ -72,7 +75,6 @@ struct desync_profile_list {
LIST_HEAD(desync_profile_list_head, desync_profile_list);
struct desync_profile_list *dp_list_add(struct desync_profile_list_head *head);
void dp_list_destroy(struct desync_profile_list_head *head);
bool dp_list_have_autohostlist(struct desync_profile_list_head *head);
struct params_s
{
@ -112,6 +114,11 @@ struct params_s
int ttl_default;
char hostlist_auto_debuglog[PATH_MAX];
// hostlist files with data for all profiles
struct hostlist_files_head hostlists;
// ipset files with data for all profiles
struct ipset_files_head ipsets;
bool tamper; // any tamper option is set
bool tamper_lim; // tamper-start or tamper-cutoff set in any profile
struct desync_profile_list_head desync_profiles;

179
tpws/pools.c
View File

@ -52,12 +52,6 @@ bool StrPoolAddStr(strpool **pp, const char *s)
{
return StrPoolAddStrLen(pp, s, strlen(s));
}
bool StrPoolAddUniqueStr(strpool **pp,const char *s)
{
if (StrPoolCheckStr(*pp,s))
return true;
return StrPoolAddStr(pp,s);
}
bool StrPoolCheckStr(strpool *p, const char *s)
{
@ -160,6 +154,93 @@ void strlist_destroy(struct str_list_head *head)
struct hostlist_file *hostlist_files_add(struct hostlist_files_head *head, const char *filename)
{
struct hostlist_file *entry = malloc(sizeof(struct hostlist_file));
if (entry)
{
if (!(entry->filename = strdup(filename)))
{
free(entry);
return false;
}
entry->mod_time=0;
entry->hostlist = NULL;
LIST_INSERT_HEAD(head, entry, next);
}
return entry;
}
static void hostlist_files_entry_destroy(struct hostlist_file *entry)
{
if (entry->filename) free(entry->filename);
StrPoolDestroy(&entry->hostlist);
free(entry);
}
void hostlist_files_destroy(struct hostlist_files_head *head)
{
struct hostlist_file *entry;
while ((entry = LIST_FIRST(head)))
{
LIST_REMOVE(entry, next);
hostlist_files_entry_destroy(entry);
}
}
struct hostlist_file *hostlist_files_search(struct hostlist_files_head *head, const char *filename)
{
struct hostlist_file *hfile;
LIST_FOREACH(hfile, head, next)
{
if (!strcmp(hfile->filename,filename))
return hfile;
}
return NULL;
}
struct hostlist_item *hostlist_collection_add(struct hostlist_collection_head *head, struct hostlist_file *hfile)
{
struct hostlist_item *entry = malloc(sizeof(struct hostlist_item));
if (entry)
{
entry->hfile = hfile;
LIST_INSERT_HEAD(head, entry, next);
}
return entry;
}
void hostlist_collection_destroy(struct hostlist_collection_head *head)
{
struct hostlist_item *entry;
while ((entry = LIST_FIRST(head)))
{
LIST_REMOVE(entry, next);
free(entry);
}
}
struct hostlist_item *hostlist_collection_search(struct hostlist_collection_head *head, const char *filename)
{
struct hostlist_item *item;
LIST_FOREACH(item, head, next)
{
if (!strcmp(item->hfile->filename,filename))
return item;
}
return NULL;
}
bool hostlist_collection_is_empty(const struct hostlist_collection_head *head)
{
const struct hostlist_item *item;
LIST_FOREACH(item, head, next)
{
if (item->hfile->hostlist)
return false;
}
return true;
}
void ipset4Destroy(ipset4 **ipset)
{
ipset4 *elem, *tmp;
@ -281,3 +362,89 @@ void ipsetPrint(ipset *ipset)
ipset4Print(ipset->ips4);
ipset6Print(ipset->ips6);
}
struct ipset_file *ipset_files_add(struct ipset_files_head *head, const char *filename)
{
struct ipset_file *entry = malloc(sizeof(struct ipset_file));
if (entry)
{
if (!(entry->filename = strdup(filename)))
{
free(entry);
return false;
}
entry->mod_time=0;
memset(&entry->ipset,0,sizeof(entry->ipset));
LIST_INSERT_HEAD(head, entry, next);
}
return entry;
}
static void ipset_files_entry_destroy(struct ipset_file *entry)
{
if (entry->filename) free(entry->filename);
ipsetDestroy(&entry->ipset);
free(entry);
}
void ipset_files_destroy(struct ipset_files_head *head)
{
struct ipset_file *entry;
while ((entry = LIST_FIRST(head)))
{
LIST_REMOVE(entry, next);
ipset_files_entry_destroy(entry);
}
}
struct ipset_file *ipset_files_search(struct ipset_files_head *head, const char *filename)
{
struct ipset_file *hfile;
LIST_FOREACH(hfile, head, next)
{
if (!strcmp(hfile->filename,filename))
return hfile;
}
return NULL;
}
struct ipset_item *ipset_collection_add(struct ipset_collection_head *head, struct ipset_file *hfile)
{
struct ipset_item *entry = malloc(sizeof(struct ipset_item));
if (entry)
{
entry->hfile = hfile;
LIST_INSERT_HEAD(head, entry, next);
}
return entry;
}
void ipset_collection_destroy(struct ipset_collection_head *head)
{
struct ipset_item *entry;
while ((entry = LIST_FIRST(head)))
{
LIST_REMOVE(entry, next);
free(entry);
}
}
struct ipset_item *ipset_collection_search(struct ipset_collection_head *head, const char *filename)
{
struct ipset_item *item;
LIST_FOREACH(item, head, next)
{
if (!strcmp(item->hfile->filename,filename))
return item;
}
return NULL;
}
bool ipset_collection_is_empty(const struct ipset_collection_head *head)
{
const struct ipset_item *item;
LIST_FOREACH(item, head, next)
{
if (!IPSET_EMPTY(&item->hfile->ipset))
return false;
}
return true;
}

49
tpws/pools.h
View File

@ -19,7 +19,6 @@ typedef struct strpool {
void StrPoolDestroy(strpool **pp);
bool StrPoolAddStr(strpool **pp,const char *s);
bool StrPoolAddUniqueStr(strpool **pp,const char *s);
bool StrPoolAddStrLen(strpool **pp,const char *s,size_t slen);
bool StrPoolCheckStr(strpool *p,const char *s);
@ -29,7 +28,6 @@ struct str_list {
};
LIST_HEAD(str_list_head, str_list);
typedef struct hostfail_pool {
char *str; /* key */
int counter; /* value */
@ -49,6 +47,30 @@ bool strlist_add(struct str_list_head *head, const char *filename);
void strlist_destroy(struct str_list_head *head);
struct hostlist_file {
char *filename;
time_t mod_time;
strpool *hostlist;
LIST_ENTRY(hostlist_file) next;
};
LIST_HEAD(hostlist_files_head, hostlist_file);
struct hostlist_file *hostlist_files_add(struct hostlist_files_head *head, const char *filename);
void hostlist_files_destroy(struct hostlist_files_head *head);
struct hostlist_file *hostlist_files_search(struct hostlist_files_head *head, const char *filename);
struct hostlist_item {
struct hostlist_file *hfile;
LIST_ENTRY(hostlist_item) next;
};
LIST_HEAD(hostlist_collection_head, hostlist_item);
struct hostlist_item *hostlist_collection_add(struct hostlist_collection_head *head, struct hostlist_file *hfile);
void hostlist_collection_destroy(struct hostlist_collection_head *head);
struct hostlist_item *hostlist_collection_search(struct hostlist_collection_head *head, const char *filename);
bool hostlist_collection_is_empty(const struct hostlist_collection_head *head);
typedef struct ipset4 {
struct cidr4 cidr; /* key */
UT_hash_handle hh; /* makes this structure hashable */
@ -85,3 +107,26 @@ void ipset6Print(ipset6 *ipset);
void ipsetDestroy(ipset *ipset);
void ipsetPrint(ipset *ipset);
struct ipset_file {
char *filename;
time_t mod_time;
ipset ipset;
LIST_ENTRY(ipset_file) next;
};
LIST_HEAD(ipset_files_head, ipset_file);
struct ipset_file *ipset_files_add(struct ipset_files_head *head, const char *filename);
void ipset_files_destroy(struct ipset_files_head *head);
struct ipset_file *ipset_files_search(struct ipset_files_head *head, const char *filename);
struct ipset_item {
struct ipset_file *hfile;
LIST_ENTRY(ipset_item) next;
};
LIST_HEAD(ipset_collection_head, ipset_item);
struct ipset_item * ipset_collection_add(struct ipset_collection_head *head, struct ipset_file *hfile);
void ipset_collection_destroy(struct ipset_collection_head *head);
struct ipset_item *ipset_collection_search(struct ipset_collection_head *head, const char *filename);
bool ipset_collection_is_empty(const struct ipset_collection_head *head);

72
tpws/tamper.c
View File

@ -24,40 +24,36 @@ static bool l7_proto_match(t_l7proto l7proto, uint32_t filter_l7)
(l7proto==TLS && (filter_l7 & L7_PROTO_TLS));
}
static bool dp_match_l3l4(struct desync_profile *dp, const struct sockaddr *dest)
{
return ((dest->sa_family==AF_INET && dp->filter_ipv4) || (dest->sa_family==AF_INET6 && dp->filter_ipv6)) &&
pf_in_range(saport(dest), &dp->pf_tcp) &&
IpsetCheck(dp, dest->sa_family==AF_INET ? &((struct sockaddr_in*)dest)->sin_addr : NULL, dest->sa_family==AF_INET6 ? &((struct sockaddr_in6*)dest)->sin6_addr : NULL);
}
static bool dp_impossible(struct desync_profile *dp, const char *hostname, t_l7proto l7proto)
{
return !PROFILE_IPSETS_EMPTY(dp) &&
((dp->filter_l7 && !l7_proto_match(l7proto, dp->filter_l7)) || (!*dp->hostlist_auto_filename && !hostname && (dp->hostlist || dp->hostlist_exclude)));
}
static bool dp_match(struct desync_profile *dp, const struct sockaddr *dest, const char *hostname, t_l7proto l7proto)
{
// impossible case, hard filter
// impossible check avoids relatively slow ipset search
if (!dp_impossible(dp,hostname,l7proto) && dp_match_l3l4(dp,dest))
{
// soft filter
if (dp->filter_l7 && !l7_proto_match(l7proto, dp->filter_l7))
return false;
if (!HostlistsReloadCheckForProfile(dp)) return false;
// autohostlist profile matching l3/l4 filter always win
if (*dp->hostlist_auto_filename) return true;
if ((dest->sa_family==AF_INET && !dp->filter_ipv4) || (dest->sa_family==AF_INET6 && !dp->filter_ipv6))
// L3 filter does not match
return false;
if (!pf_in_range(saport(dest), &dp->pf_tcp))
// L4 filter does not match
return false;
if (dp->filter_l7 && !l7_proto_match(l7proto, dp->filter_l7))
// L7 filter does not match
return false;
if (!dp->hostlist_auto && !hostname && !PROFILE_HOSTLISTS_EMPTY(dp))
// avoid cpu consuming ipset check. profile cannot win if regular hostlists are present without auto hostlist and hostname is unknown.
return false;
if (!IpsetCheck(dp, dest->sa_family==AF_INET ? &((struct sockaddr_in*)dest)->sin_addr : NULL, dest->sa_family==AF_INET6 ? &((struct sockaddr_in6*)dest)->sin6_addr : NULL))
// target ip does not match
return false;
// autohostlist profile matching l3/l4 filter always win
if (dp->hostlist_auto) return true;
if (PROFILE_HOSTLISTS_EMPTY(dp))
// profile without hostlist filter wins
return true;
else if (hostname)
// without known hostname first profile matching l3/l4 filter and without hostlist filter wins
return HostlistCheck(dp, hostname, NULL, true);
if (dp->hostlist || dp->hostlist_exclude)
{
// without known hostname first profile matching l3/l4 filter and without hostlist filter wins
if (hostname)
return HostlistCheck(dp, hostname, NULL);
}
else
// profile without hostlist filter wins
return true;
}
return false;
}
static struct desync_profile *dp_find(struct desync_profile_list_head *head, const struct sockaddr *dest, const char *hostname, t_l7proto l7proto)
@ -181,10 +177,10 @@ void tamper_out(t_ctrack *ctrack, const struct sockaddr *dest, uint8_t *segment,
VPRINT("desync profile changed by revealed l7 protocol or hostname !\n");
}
if (bDiscoveredHostname && *ctrack->dp->hostlist_auto_filename)
if (bDiscoveredHostname && ctrack->dp->hostlist_auto)
{
bool bHostExcluded;
if (!HostlistCheck(ctrack->dp, Host, &bHostExcluded))
if (!HostlistCheck(ctrack->dp, Host, &bHostExcluded, false))
{
ctrack->b_ah_check = !bHostExcluded;
VPRINT("Not acting on this request\n");
@ -415,21 +411,21 @@ static void auto_hostlist_failed(struct desync_profile *dp, const char *hostname
VPRINT("auto hostlist (profile %d) : rechecking %s to avoid duplicates\n", dp->n, hostname);
bool bExcluded=false;
if (!HostlistCheck(dp, hostname, &bExcluded) && !bExcluded)
if (!HostlistCheck(dp, hostname, &bExcluded, false) && !bExcluded)
{
VPRINT("auto hostlist (profile %d) : adding %s to %s\n", dp->n, hostname, dp->hostlist_auto_filename);
HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : client %s : proto %s : adding to %s", hostname, dp->n, client_ip_port, l7proto_str(l7proto), dp->hostlist_auto_filename);
if (!StrPoolAddStr(&dp->hostlist, hostname))
VPRINT("auto hostlist (profile %d) : adding %s to %s\n", dp->n, hostname, dp->hostlist_auto->filename);
HOSTLIST_DEBUGLOG_APPEND("%s : profile %d : client %s : proto %s : adding to %s", hostname, dp->n, client_ip_port, l7proto_str(l7proto), dp->hostlist_auto->filename);
if (!StrPoolAddStr(&dp->hostlist_auto->hostlist, hostname))
{
DLOG_ERR("StrPoolAddStr out of memory\n");
return;
}
if (!append_to_list_file(dp->hostlist_auto_filename, hostname))
if (!append_to_list_file(dp->hostlist_auto->filename, hostname))
{
DLOG_PERROR("write to auto hostlist:");
return;
}
dp->hostlist_auto_mod_time = file_mod_time(dp->hostlist_auto_filename);
dp->hostlist_auto->mod_time = file_mod_time(dp->hostlist_auto->filename);
}
else
{

73
tpws/tpws.c
View File

@ -43,25 +43,9 @@
struct params_s params;
bool bHup = false;
static void onhup(int sig)
{
printf("HUP received !\n");
printf("Will reload hostlists and ipsets on next request (if any)\n");
bHup = true;
}
// should be called in normal execution
void dohup(void)
{
if (bHup)
{
if (!LoadIncludeHostLists() || !LoadExcludeHostLists() || !LoadIncludeIpsets() || !LoadExcludeIpsets())
{
// what will we do without hostlist or ipset ?? sure, gonna die
exit(1);
}
bHup = false;
}
}
static void onusr2(int sig)
@ -230,6 +214,9 @@ static void exithelp(void)
static void cleanup_params(void)
{
dp_list_destroy(&params.desync_profiles);
hostlist_files_destroy(&params.hostlists);
ipset_files_destroy(&params.ipsets);
}
static void exithelp_clean(void)
{
@ -367,6 +354,10 @@ void parse_params(int argc, char *argv[])
#if defined(__OpenBSD__) || defined(__APPLE__)
params.pf_enable = true; // OpenBSD and MacOS have no other choice
#endif
LIST_INIT(&params.hostlists);
LIST_INIT(&params.ipsets);
if (can_drop_root())
{
params.uid = params.gid = 0x7FFFFFFF; // default uid:gid
@ -726,29 +717,29 @@ void parse_params(int argc, char *argv[])
params.tamper = true;
break;
case 36: /* hostlist */
if (!strlist_add(&dp->hostlist_files, optarg))
if (!RegisterHostlist(dp, false, optarg))
{
DLOG_ERR("strlist_add failed\n");
DLOG_ERR("failed to register hostlist '%s'\n", optarg);
exit_clean(1);
}
params.tamper = true;
break;
case 37: /* hostlist-exclude */
if (!strlist_add(&dp->hostlist_exclude_files, optarg))
if (!RegisterHostlist(dp, true, optarg))
{
DLOG_ERR("strlist_add failed\n");
DLOG_ERR("failed to register hostlist '%s'\n", optarg);
exit_clean(1);
}
params.tamper = true;
break;
case 38: /* hostlist-auto */
if (*dp->hostlist_auto_filename)
if (dp->hostlist_auto)
{
DLOG_ERR("only one auto hostlist per profile is supported\n");
exit_clean(1);
}
{
FILE *F = fopen(optarg,"a+t");
FILE *F = fopen(optarg,"a+b");
if (!F)
{
DLOG_ERR("cannot create %s\n", optarg);
@ -764,13 +755,11 @@ void parse_params(int argc, char *argv[])
if (params.droproot && chown(optarg, params.uid, -1))
DLOG_ERR("could not chown %s. auto hostlist file may not be writable after privilege drop\n", optarg);
}
if (!strlist_add(&dp->hostlist_files, optarg))
if (!(dp->hostlist_auto=RegisterHostlist(dp, false, optarg)))
{
DLOG_ERR("strlist_add failed\n");
DLOG_ERR("failed to register hostlist '%s'\n", optarg);
exit_clean(1);
}
strncpy(dp->hostlist_auto_filename, optarg, sizeof(dp->hostlist_auto_filename));
dp->hostlist_auto_filename[sizeof(dp->hostlist_auto_filename) - 1] = '\0';
params.tamper = true; // need to detect blocks and update autohostlist. cannot just slice.
break;
case 39: /* hostlist-auto-fail-threshold */
@ -979,17 +968,17 @@ void parse_params(int argc, char *argv[])
}
break;
case 60: /* ipset */
if (!strlist_add(&dp->ipset_files, optarg))
if (!RegisterIpset(dp, false, optarg))
{
DLOG_ERR("strlist_add failed\n");
DLOG_ERR("failed to register ipset '%s'\n", optarg);
exit_clean(1);
}
params.tamper = true;
break;
case 61: /* ipset-exclude */
if (!strlist_add(&dp->ipset_exclude_files, optarg))
if (!RegisterIpset(dp, true, optarg))
{
DLOG_ERR("strlist_add failed\n");
DLOG_ERR("failed to register ipset '%s'\n", optarg);
exit_clean(1);
}
params.tamper = true;
@ -1062,7 +1051,6 @@ void parse_params(int argc, char *argv[])
dp = &dpl->dp;
if (dp->split_tls==tlspos_none && dp->split_pos) dp->split_tls=tlspos_pos;
if (dp->split_http_req==httpreqpos_none && dp->split_pos) dp->split_http_req=httpreqpos_pos;
if (*dp->hostlist_auto_filename) dp->hostlist_auto_mod_time = file_mod_time(dp->hostlist_auto_filename);
if (params.skip_nodelay && (dp->split_tls || dp->split_http_req || dp->split_pos))
{
DLOG_ERR("Cannot split with --skip-nodelay\n");
@ -1070,26 +1058,21 @@ void parse_params(int argc, char *argv[])
}
}
if (!LoadIncludeHostLists())
if (!LoadAllHostLists())
{
DLOG_ERR("Include hostlist load failed\n");
DLOG_ERR("hostlists load failed\n");
exit_clean(1);
}
if (!LoadExcludeHostLists())
if (!LoadAllIpsets())
{
DLOG_ERR("Exclude hostlist load failed\n");
exit_clean(1);
}
if (!LoadIncludeIpsets())
{
DLOG_ERR("Include ipset load failed\n");
exit_clean(1);
}
if (!LoadExcludeIpsets())
{
DLOG_ERR("Exclude ipset load failed\n");
DLOG_ERR("ipset load failed\n");
exit_clean(1);
}
VPRINT("\nlists summary:\n");
HostlistsDebug();
IpsetsDebug();
VPRINT("\n");
}

2
tpws/tpws_conn.c
View File

@ -1469,8 +1469,6 @@ int event_loop(const int *listen_fd, size_t listen_fd_ct)
break;
}
dohup();
for (i = 0; i < num_events; i++)
{
conn = (tproxy_conn_t*)events[i].data.ptr;