ustor.c

// James William Flecher [github.com/mrbid] ~2018
// Not for public release.
///////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////
//////////////////////////////////////////////
/////////////////////////////
///////////////
////////

#include "ustor.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include "sha1.h"
#include "crc64.h"

///////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////
//////////////////////////////////////////////
/////////////////////////////
///////////////
////////
///
//
//
//
/*
.   SHA1 hashing
*/

//SHA1 Hash Func
void sre1(sre *r, const char* str)
{
    SHA1(r->digest, str, strlen(str));
}

//SHA1 Hash Comparison Func
int sre_com(const sre *d1, const sre *d2)
{
    if(memcmp(d1->digest, d2->digest, HASH_LEN) == 0)
        return 1; //Are the same hashes
    return 0; //Not the same hashes
}

//SHA1 Hash Copy Func
void sre_copy(sre *d, const sre *src)
{
    memcpy(d->digest, src->digest, HASH_LEN);
}

//SHA1 Hash Null Func
void sre_null(sre *d)
{
    memset(d->digest, 0, HASH_LEN);
}

//SHA1 Hash IsNull Func
int sre_isnull(sre *d)
{
    const static sre snull = {{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}};
    if(memcmp(d->digest, &snull, HASH_LEN) == 0)
        return 1; //Are the same hashes
    return 0; //Not the same hashes
}

///////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////
//////////////////////////////////////////////
/////////////////////////////
///////////////
////////
///
//
//
//
/*
.
.
.
.
. ~ Unique Capping Code,
.   an array of impressions per site,
.   crc32 hashing,
.   fixed allocated memory size.
*/

//333331, 888887, 1258297, 3333331
uint MAX_SITES = 333331; //1258297 (3gb)
#define MAX_COLS 128

//Impressions are split into site domain buckets
struct site
{
    sre col[MAX_COLS];
    uint write_head;
    uint expire_epoch;
};

//Our buckets
struct site *sites;

//Logs
ulong blocked=0, allowed=0, rejected=0, collisions=0;

//Save dump
void savemem()
{
    FILE* f = fopen("state/ustor_state", "w");
    if(f)
    {
        fwrite(sites, sizeof(struct site), MAX_SITES, f);
        fclose(f);
    }
}

//Load dump
int loadmem()
{
    FILE* f = fopen("state/ustor_state", "r");
    if(f)
    {
        fseek(f, 0L, SEEK_END);
        const long len = ftell(f);
        rewind(f);

        if(len != MAX_SITES*sizeof(struct site))
            return 0;

        printf("Loading USTOR State ...\n");
        fread(sites, sizeof(struct site), MAX_SITES, f);
        fclose(f);
        return 1;
    }
    return 0;
}

//Initialise USTOR buffers
int init_ustor(const unsigned int msi)
{
    if(msi != 0)
    {
        MAX_SITES = msi;
        sites = malloc(MAX_SITES * sizeof(struct site));
        if(sites == NULL)
        {
            printf("USTOR Failed to allocate %lu bytes (%lu mb)\n", MAX_SITES*sizeof(struct site), (MAX_SITES*sizeof(struct site)) / 1024 / 1024);
            return 0;
        }
        printf("USTOR allocated %lu bytes (%lu mb)\n", MAX_SITES*sizeof(struct site), (MAX_SITES*sizeof(struct site)) / 1024 / 1024);
        return 1;
    }

    MAX_SITES = 3333331;
    sites = malloc(MAX_SITES * sizeof(struct site));
    if(sites == NULL)
    {
        printf("USTOR Failed to allocate %lu bytes (%lu mb)\n", MAX_SITES*sizeof(struct site), (MAX_SITES*sizeof(struct site)) / 1024 / 1024);
        MAX_SITES = 1258297;
        sites = malloc(MAX_SITES * sizeof(struct site));
        if(sites == NULL)
        {
            printf("USTOR Failed to allocate %lu bytes (%lu mb)\n", MAX_SITES*sizeof(struct site), (MAX_SITES*sizeof(struct site)) / 1024 / 1024);
            MAX_SITES = 888887;
            sites = malloc(MAX_SITES * sizeof(struct site));
            if(sites == NULL)
            {
                printf("USTOR Failed to allocate %lu bytes (%lu mb)\n", MAX_SITES*sizeof(struct site), (MAX_SITES*sizeof(struct site)) / 1024 / 1024);
                MAX_SITES = 333331;
                sites = malloc(MAX_SITES * sizeof(struct site));
                if(sites == NULL)
                {
                    printf("USTOR Failed to allocate %lu bytes (%lu mb)\n", MAX_SITES*sizeof(struct site), (MAX_SITES*sizeof(struct site)) / 1024 / 1024);
                    MAX_SITES = 233327;
                    sites = malloc(MAX_SITES * sizeof(struct site));
                    if(sites == NULL)
                    {
                        printf("USTOR Failed to allocate %lu bytes (%lu mb)\n", MAX_SITES*sizeof(struct site), (MAX_SITES*sizeof(struct site)) / 1024 / 1024);
                        MAX_SITES = 144451;
                        sites = malloc(MAX_SITES * sizeof(struct site));
                        if(sites == NULL)
                        {
                            perror("USTOR Failed to allocate memory for Unique Capping");
                            return 0;
                        }
                    }
                }
            }
        }
    }
    printf("USTOR allocated %lu bytes (%lu mb)\n", MAX_SITES*sizeof(struct site), (MAX_SITES*sizeof(struct site)) / 1024 / 1024);

    for(int i = 0; i < MAX_SITES; ++i)
    {
        memset(sites[i].col, 0, sizeof(sre)*MAX_COLS); //Empty buffer
        sites[i].write_head = 0;
        sites[i].expire_epoch = 0;
    }

    return 1;
}

//Reset USTOR
void reset_ustor()
{
    //I could just memset the whole block but, it's not critical
    for(int i = 0; i < MAX_SITES; ++i)
    {
        memset(sites[i].col, 0, sizeof(sre)*MAX_COLS); //Empty buffer
        sites[i].write_head = 0;
        sites[i].expire_epoch = 0;
    }
}

//Check against all idfa in memory for a match, if it's not there, add it,
int has_idfa(const uint64_t idfa, const sre* uid) //Pub
{
    //Find site index
    const uint site_index = idfa % MAX_SITES;

    //Reset if expire_epoch dictates this site bucket is expired 
    if(time(0) >= sites[site_index].expire_epoch)
    {
        memset(sites[site_index].col, 0, sizeof(sre)*MAX_COLS); //Empty buffer
        sites[site_index].write_head = 0;
        sites[site_index].expire_epoch = 0;

        //fprintf(stderr, "%i: reached it's expirary after $u hours.\n", cid_index, ((time(0) - campaigns[cid_index].sites[site_index].last_epoch) / 60) / 60 );
    }

    //Check the ranges
    for(int i=0; i < MAX_COLS; i++)
    {
        if(sre_isnull(&sites[site_index].col[i]) == 1)
            break;
        else if(sre_com(&sites[site_index].col[i], uid) == 1)
        {
            rejected++;
            return 1;
        }
    }

    //Stats
    allowed++;

    //add_idfa_priv(site_index, idfa); //No idfa? Add it
    return 0; //NO IFDA :(
}

//Set's ifa,
void add_idfa(const uint64_t idfa, const sre* uid, const uint expire_seconds) //Pub
{
    //Find site index
    const uint site_index = idfa % MAX_SITES;

    //Reset if expire_epoch dictates this site bucket is expired 
    if(time(0) >= sites[site_index].expire_epoch)
    {
        memset(sites[site_index].col, 0, sizeof(sre)*MAX_COLS); //Empty buffer
        sites[site_index].write_head = 0;
        sites[site_index].expire_epoch = time(0)+expire_seconds;

        //fprintf(stderr, "%i: reached it's expirary after $u hours.\n", cid_index, ((time(0) - campaigns[cid_index].sites[site_index].last_epoch) / 60) / 60 );
    }

    //Write the new idfa
    sre_copy(&sites[site_index].col[sites[site_index].write_head], uid);

    //Increment blocked
    blocked++;

    //Increment write head
    sites[site_index].write_head++;
    if(sites[site_index].write_head >= MAX_COLS)
    {
        sites[site_index].write_head = 0; //Replaces oldest on mac limit
        printf("\n%u Unique Bucket Peak\n\n", site_index);
        collisions++;
    }
}