Skip to content

added equi-depth SAX breakpoints for all approaches. Gaussian by defa… #64

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sophisid wants to merge 1 commit into
base: main
Choose a base branch
from
Open

added equi-depth SAX breakpoints for all approaches. Gaussian by defa… #64

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
*.part
*.full
isax_file.sax
daisy_index_nodes/

# Build directories
build/
Expand Down
8 changes: 4 additions & 4 deletions lib/algos/DumpyOS.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -420,13 +420,13 @@ static void get_value_range(int sax_val, int bc, double* lb, double* ub) {
int offset = ((cardinality - 1) * (cardinality - 2)) / 2;
if (sax_val == 0) {
*lb = -std::numeric_limits<double>::max();
*ub = sax_breakpoints[offset];
*ub = daisy_active_breakpoints[offset];
} else if (sax_val == cardinality - 1) {
*lb = sax_breakpoints[offset + sax_val - 1];
*lb = daisy_active_breakpoints[offset + sax_val - 1];
*ub = std::numeric_limits<double>::max();
} else {
*lb = sax_breakpoints[offset + sax_val - 1];
*ub = sax_breakpoints[offset + sax_val];
*lb = daisy_active_breakpoints[offset + sax_val - 1];
*ub = daisy_active_breakpoints[offset + sax_val];
}
}

Expand Down
9 changes: 8 additions & 1 deletion lib/algos/Fresh.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -899,7 +899,12 @@ void Fresh::buildIndex(DataSource *data_source)
this->tight_bound,
0,
1,
1);
1,
this->bp_mode);

// Equi-depth breakpoints (no-op in Gaussian mode), installed as active.
compute_equidepth_breakpoints(this->index_settings, this->database, this->n_database);
activateBreakpoints();

this->index = isax_index_init_inmemory(this->index_settings);
isax_index *index = this->index;
Expand Down Expand Up @@ -1470,6 +1475,7 @@ void Fresh::searchIndex(const float *query, idx_t n_query, const SearchConfig &c
std::vector<std::vector<idx_t>> &I,
std::vector<std::vector<float>> &D)
{
activateBreakpoints();
if (config.type == QueryType::TOP_K)
{
SimilaritySearchAlgorithm::searchIndex(query, n_query, config, I, D);
Expand Down Expand Up @@ -1513,6 +1519,7 @@ void Fresh::searchIndex(const float *query, idx_t n_query, const SearchConfig &c

void Fresh::searchIndex(const float *query, idx_t n_query, idx_t k, idx_t *I, float *D)
{
activateBreakpoints();
if (this->distance_type == DistanceType::L2_SQUARED)
searchIndexL2Squared(query, n_query, k, I, D);
else if (this->distance_type == DistanceType::DTW)
Expand Down
9 changes: 8 additions & 1 deletion lib/algos/LbBruteforce.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,12 @@ namespace daisy
this->tight_bound,
0,
1,
1);
1,
this->bp_mode);

// Equi-depth breakpoints (no-op in Gaussian mode), installed as active.
compute_equidepth_breakpoints(this->index_settings, this->database, this->n_database);
activateBreakpoints();

this->index = isax_index_init_inmemory(this->index_settings);

Expand All @@ -105,6 +110,7 @@ namespace daisy

void LbBruteforce::searchIndex(const float *query, const idx_t n_query, const idx_t k, idx_t *I, float *D)
{
activateBreakpoints();
if (this->distance_type == DistanceType::L2_SQUARED)
{
searchIndexL2Squared(query, n_query, k, I, D);
Expand Down Expand Up @@ -287,6 +293,7 @@ namespace daisy
std::vector<std::vector<idx_t>> &I,
std::vector<std::vector<float>> &D)
{
activateBreakpoints();
if (config.type == QueryType::TOP_K) {
SimilaritySearchAlgorithm::searchIndex(query, n_query, config, I, D);
return;
Expand Down
9 changes: 8 additions & 1 deletion lib/algos/Messi.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -794,7 +794,12 @@ namespace daisy
this->tight_bound,
0,
1,
1);
1,
this->bp_mode);

// Equi-depth breakpoints (no-op in Gaussian mode), installed as active.
compute_equidepth_breakpoints(this->index_settings, this->database, this->n_database);
activateBreakpoints();

this->index = isax_index_init_inmemory(this->index_settings);
isax_index *index = this->index;
Expand Down Expand Up @@ -1074,6 +1079,7 @@ namespace daisy
std::vector<std::vector<idx_t>> &I,
std::vector<std::vector<float>> &D)
{
activateBreakpoints();
if (config.type == QueryType::TOP_K)
{
SimilaritySearchAlgorithm::searchIndex(query, n_query, config, I, D);
Expand Down Expand Up @@ -1117,6 +1123,7 @@ namespace daisy

void Messi::searchIndex(const float *query, const idx_t n_query, const idx_t k, idx_t *I, float *D)
{
activateBreakpoints();
if (this->distance_type == DistanceType::L2_SQUARED)
{
searchIndexL2Squared(query, n_query, k, I, D);
Expand Down
28 changes: 24 additions & 4 deletions lib/algos/ParIS.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,28 @@ namespace daisy
this->flush_limit,
this->initial_fbl_size,
this->total_loaded_leaves,
this->tight_bound,
0,
1,
0);
this->tight_bound,
0,
1,
0,
this->bp_mode);

// Equi-depth breakpoints. ParIS streams from a raw float32 file, so sample leading
// rows from it to derive the distribution.
if (this->bp_mode == BP_EQUIDEPTH && this->n_database > 0 && this->dim > 0)
{
size_t cap_rows = (size_t)2000000 / (size_t)std::max(1, this->paa_segments);
size_t sample_rows = std::min((size_t)this->n_database, std::max((size_t)1, cap_rows));
std::vector<float> sample((size_t)sample_rows * (size_t)this->dim);
FILE *sf = fopen(filename, "rb");
if (sf)
{
size_t got = fread(sample.data(), sizeof(float), sample.size(), sf);
fclose(sf);
compute_equidepth_breakpoints(this->index_settings, sample.data(), got / (size_t)this->dim);
}
}
activateBreakpoints();

this->index = isax_index_init(this->index_settings);
isax_index *index = this->index;
Expand Down Expand Up @@ -151,6 +169,7 @@ namespace daisy

void ParIS::searchIndex(const float *query, const idx_t n_query, const idx_t k, idx_t *I, float *D)
{
activateBreakpoints();
if (this->distance_type == DistanceType::L2_SQUARED)
{
searchIndexL2Squared(query, n_query, k, I, D);
Expand Down Expand Up @@ -1366,6 +1385,7 @@ namespace daisy
std::vector<std::vector<idx_t>> &I,
std::vector<std::vector<float>> &D)
{
activateBreakpoints();
if (config.type == QueryType::TOP_K) {
SimilaritySearchAlgorithm::searchIndex(query, n_query, config, I, D);
return;
Expand Down
16 changes: 16 additions & 0 deletions lib/algos/SimilaritySearchAlgorithm.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ namespace daisy
float minimum_distance = FLT_MAX;
int min_checked_leaves = -1;

breakpoint_mode bp_mode = BP_GAUSSIAN;

isax_index_settings *index_settings = nullptr;
isax_index *index = nullptr;
sax_type **db_sax_representations = nullptr;
Expand All @@ -68,6 +70,20 @@ namespace daisy
idx_t getDim() const { return dim; }
isax_index *getIndex() const { return index; }

// 0 = Gaussian (default), 1 = equi-depth. Set before buildIndex().
void setBreakpointMode(int mode) { bp_mode = (mode == 1) ? BP_EQUIDEPTH : BP_GAUSSIAN; }
int getBreakpointMode() const { return (int)bp_mode; }

protected:
// Install this index's breakpoints globally; call at build and at each search entry.
void activateBreakpoints()
{
if (index_settings)
set_active_breakpoints(index_settings->breakpoints, index_settings->breakpoints_max);
}

public:

virtual void buildIndex(DataSource *data_source) = 0;

virtual void buildIndex(float *database, idx_t n_database, idx_t dim)
Expand Down
16 changes: 8 additions & 8 deletions lib/algos/Sing.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,17 +105,17 @@ namespace daisy
__m256i lower_juge_nzerov_1 = _mm256_andnot_si256(lower_juge_zerov_1, vectorsignbit);
__m256 minvalv = _mm256_set1_ps((float)min_val);

__m256 lsax_breakpoints_shiftv_0 = _mm256_i32gather_ps(sax_breakpointsnew3, region_lowerv_0, 4);
__m256 lsax_breakpoints_shiftv_1 = _mm256_i32gather_ps(sax_breakpointsnew3, region_lowerv_1, 4);
__m256 lsax_breakpoints_shiftv_0 = _mm256_i32gather_ps(daisy_active_breakpoints_max, region_lowerv_0, 4);
__m256 lsax_breakpoints_shiftv_1 = _mm256_i32gather_ps(daisy_active_breakpoints_max, region_lowerv_1, 4);
__m256 breakpoint_lowerv_0 = (__m256)_mm256_castsi256_ps(_mm256_or_si256(
_mm256_and_si256(lower_juge_zerov_0, _mm256_castps_si256(minvalv)),
_mm256_and_si256(lower_juge_nzerov_0, _mm256_castps_si256(lsax_breakpoints_shiftv_0))));
__m256 breakpoint_lowerv_1 = (__m256)_mm256_castsi256_ps(_mm256_or_si256(
_mm256_and_si256(lower_juge_zerov_1, _mm256_castps_si256(minvalv)),
_mm256_and_si256(lower_juge_nzerov_1, _mm256_castps_si256(lsax_breakpoints_shiftv_1))));

__m256 usax_breakpoints_shiftv_0 = _mm256_i32gather_ps(sax_breakpointsnew3, region_upperv_0, 4);
__m256 usax_breakpoints_shiftv_1 = _mm256_i32gather_ps(sax_breakpointsnew3, region_upperv_1, 4);
__m256 usax_breakpoints_shiftv_0 = _mm256_i32gather_ps(daisy_active_breakpoints_max, region_upperv_0, 4);
__m256 usax_breakpoints_shiftv_1 = _mm256_i32gather_ps(daisy_active_breakpoints_max, region_upperv_1, 4);
__m256i upper_juge_maxv_0 = _mm256_cmpeq_epi32(region_upperv_0, _mm256_set1_epi32(max_cardinality - 1));
__m256i upper_juge_maxv_1 = _mm256_cmpeq_epi32(region_upperv_1, _mm256_set1_epi32(max_cardinality - 1));
__m256i upper_juge_nmaxv_0 = _mm256_andnot_si256(upper_juge_maxv_0, vectorsignbit);
Expand Down Expand Up @@ -267,15 +267,15 @@ namespace daisy
}
else
{
breakpoint_lower = sax_breakpointsnew3[region_lower];
breakpoint_lower = daisy_active_breakpoints_max[region_lower];
}
if (region_upper == (sax_type)(max_cardinality - 1))
{
breakpoint_upper = (float)max_val;
}
else
{
breakpoint_upper = sax_breakpointsnew3[region_upper + 1];
breakpoint_upper = daisy_active_breakpoints_max[region_upper + 1];
}

if (breakpoint_lower > paa[i])
Expand Down Expand Up @@ -471,15 +471,15 @@ namespace daisy
}
else
{
breakpoint_lower = sax_breakpointsnew3[region_lower - 1];
breakpoint_lower = daisy_active_breakpoints_max[region_lower - 1];
}
if (region_upper == (sax_type)(max_cardinality - 1))
{
breakpoint_upper = (float)max_val;
}
else
{
breakpoint_upper = sax_breakpointsnew3[region_upper];
breakpoint_upper = daisy_active_breakpoints_max[region_upper];
}

(void)mask;
Expand Down
8 changes: 7 additions & 1 deletion lib/algos/Sofa.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -867,6 +867,7 @@ void Sofa::searchIndexDTW(const float *query, idx_t n_query, idx_t k, idx_t *I,

void Sofa::searchIndex(const float *query, const idx_t n_query, const idx_t k, idx_t *I, float *D)
{
activateBreakpoints();
if (this->distance_type == DistanceType::L2_SQUARED)
searchIndexL2Squared(query, n_query, k, I, D);
else if (this->distance_type == DistanceType::DTW)
Expand All @@ -879,6 +880,7 @@ void Sofa::searchIndex(const float *query, idx_t n_query, const SearchConfig &co
std::vector<std::vector<idx_t>> &I,
std::vector<std::vector<float>> &D)
{
activateBreakpoints();
if (config.type == QueryType::TOP_K) {
SimilaritySearchAlgorithm::searchIndex(query, n_query, config, I, D);
return;
Expand Down Expand Up @@ -1047,7 +1049,11 @@ void Sofa::buildIndex(DataSource *data_source)
this->leaf_size, this->min_leaf_size,
this->initial_lbl_size, this->flush_limit,
this->initial_fbl_size, this->total_loaded_leaves,
this->tight_bound, 0, 1, 1);
this->tight_bound, 0, 1, 1, this->bp_mode);

// Equi-depth breakpoints (no-op in Gaussian mode), installed as active.
compute_equidepth_breakpoints(this->index_settings, this->database, this->n_database);
activateBreakpoints();

this->index = isax_index_init_inmemory(this->index_settings);
isax_index *index = this->index;
Expand Down
Loading
Loading