Skip to content

Commit 3a8d259

Browse files
abnegateabnegate
authored
Merge pull request #790 from utopia-php/fix-upsert-schemaless
2 parents e491ed2 + 28b4f27 commit 3a8d259

2 files changed

Lines changed: 337 additions & 0 deletions

File tree

‎src/Database/Adapter/Mongo.php‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1562,6 +1562,7 @@ public function upsertDocuments(Document $collection, string $attribute, array $
15621562
$operations = [];
15631563
foreach ($changes as $change) {
15641564
$document = $change->getNew();
1565+
$oldDocument = $change->getOld();
15651566
$attributes = $document->getAttributes();
15661567
$attributes['_uid'] = $document->getId();
15671568
$attributes['_createdAt'] = $document['$createdAt'];
@@ -1587,6 +1588,9 @@ public function upsertDocuments(Document $collection, string $attribute, array $
15871588

15881589
unset($record['_id']); // Don't update _id
15891590

1591+
// Get fields to unset for schemaless mode
1592+
$unsetFields = $this->getUpsertAttributeRemovals($oldDocument, $document, $record);
1593+
15901594
if (!empty($attribute)) {
15911595
// Get the attribute value before removing it from $set
15921596
$attributeValue = $record[$attribute] ?? 0;
@@ -1595,17 +1599,28 @@ public function upsertDocuments(Document $collection, string $attribute, array $
15951599
// it is requierd to mimic the behaver of SQL on duplicate key update
15961600
unset($record[$attribute]);
15971601

1602+
// Also remove from unset if it was there
1603+
unset($unsetFields[$attribute]);
1604+
15981605
// Increment the specific attribute and update all other fields
15991606
$update = [
16001607
'$inc' => [$attribute => $attributeValue],
16011608
'$set' => $record
16021609
];
1610+
1611+
if (!empty($unsetFields)) {
1612+
$update['$unset'] = $unsetFields;
1613+
}
16031614
} else {
16041615
// Update all fields
16051616
$update = [
16061617
'$set' => $record
16071618
];
16081619

1620+
if (!empty($unsetFields)) {
1621+
$update['$unset'] = $unsetFields;
1622+
}
1623+
16091624
// Add UUID7 _id for new documents in upsert operations
16101625
if (empty($document->getSequence())) {
16111626
$update['$setOnInsert'] = [
@@ -1634,6 +1649,43 @@ public function upsertDocuments(Document $collection, string $attribute, array $
16341649
return \array_map(fn ($change) => $change->getNew(), $changes);
16351650
}
16361651

1652+
/**
1653+
* Get fields to unset for schemaless upsert operations
1654+
*
1655+
* @param Document $oldDocument
1656+
* @param Document $newDocument
1657+
* @param array<string, mixed> $record
1658+
* @return array<string, string>
1659+
*/
1660+
private function getUpsertAttributeRemovals(Document $oldDocument, Document $newDocument, array $record): array
1661+
{
1662+
$unsetFields = [];
1663+
1664+
if ($this->getSupportForAttributes() || $oldDocument->isEmpty()) {
1665+
return $unsetFields;
1666+
}
1667+
1668+
$oldUserAttributes = $oldDocument->getAttributes();
1669+
$newUserAttributes = $newDocument->getAttributes();
1670+
1671+
$protectedFields = ['_uid', '_id', '_createdAt', '_updatedAt', '_permissions', '_tenant'];
1672+
1673+
foreach ($oldUserAttributes as $originalKey => $originalValue) {
1674+
if (in_array($originalKey, $protectedFields) || array_key_exists($originalKey, $newUserAttributes)) {
1675+
continue;
1676+
}
1677+
1678+
$transformed = $this->replaceChars('$', '_', [$originalKey => $originalValue]);
1679+
$dbKey = array_key_first($transformed);
1680+
1681+
if ($dbKey && !array_key_exists($dbKey, $record) && !in_array($dbKey, $protectedFields)) {
1682+
$unsetFields[$dbKey] = '';
1683+
}
1684+
}
1685+
1686+
return $unsetFields;
1687+
}
1688+
16371689
/**
16381690
* Get sequences for documents that were created
16391691
*

‎tests/e2e/Adapter/Scopes/SchemalessTests.php‎

Lines changed: 285 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1865,4 +1865,289 @@ public function testSchemalessNestedObjectAttributeQueries(): void
18651865

18661866
$database->deleteCollection($col);
18671867
}
1868+
1869+
public function testUpsertFieldRemoval(): void
1870+
{
1871+
/** @var Database $database */
1872+
$database = $this->getDatabase();
1873+
1874+
if ($database->getAdapter()->getSupportForAttributes()) {
1875+
$this->markTestSkipped('Adapter supports attributes (schemaful mode). Field removal in upsert is tested in schemaful tests.');
1876+
}
1877+
1878+
$collectionName = ID::unique();
1879+
$database->createCollection($collectionName, permissions: [
1880+
Permission::create(Role::any()),
1881+
Permission::read(Role::any()),
1882+
Permission::update(Role::any()),
1883+
Permission::delete(Role::any()),
1884+
]);
1885+
1886+
$permissions = [
1887+
Permission::read(Role::any()),
1888+
Permission::create(Role::any()),
1889+
Permission::update(Role::any()),
1890+
Permission::delete(Role::any()),
1891+
];
1892+
1893+
// Test 1: Basic field removal with upsertDocument
1894+
// Create a document with multiple fields
1895+
$doc1 = $database->createDocument($collectionName, new Document([
1896+
'$id' => 'doc1',
1897+
'$permissions' => $permissions,
1898+
'title' => 'Original Title',
1899+
'description' => 'Original Description',
1900+
'category' => 'tech',
1901+
'tags' => ['php', 'mongodb'],
1902+
'metadata' => [
1903+
'author' => 'John Doe',
1904+
'version' => 1
1905+
]
1906+
]));
1907+
1908+
$this->assertEquals('Original Title', $doc1->getAttribute('title'));
1909+
$this->assertEquals('Original Description', $doc1->getAttribute('description'));
1910+
$this->assertEquals('tech', $doc1->getAttribute('category'));
1911+
$this->assertArrayHasKey('tags', $doc1->getArrayCopy());
1912+
$this->assertArrayHasKey('metadata', $doc1->getArrayCopy());
1913+
1914+
// Upsert with fewer fields - removed fields should be deleted
1915+
$upserted = $database->upsertDocument($collectionName, new Document([
1916+
'$id' => 'doc1',
1917+
'$permissions' => $permissions,
1918+
'title' => 'Updated Title',
1919+
'category' => 'science',
1920+
// description, tags, and metadata are removed
1921+
]));
1922+
1923+
$this->assertEquals('Updated Title', $upserted->getAttribute('title'));
1924+
$this->assertEquals('science', $upserted->getAttribute('category'));
1925+
1926+
// Verify removed fields are actually deleted
1927+
$retrieved = $database->getDocument($collectionName, 'doc1');
1928+
$this->assertEquals('Updated Title', $retrieved->getAttribute('title'));
1929+
$this->assertEquals('science', $retrieved->getAttribute('category'));
1930+
$this->assertArrayNotHasKey('description', $retrieved->getArrayCopy());
1931+
$this->assertArrayNotHasKey('tags', $retrieved->getArrayCopy());
1932+
$this->assertArrayNotHasKey('metadata', $retrieved->getArrayCopy());
1933+
1934+
// Test 2: Remove all custom fields except one
1935+
$doc2 = $database->createDocument($collectionName, new Document([
1936+
'$id' => 'doc2',
1937+
'$permissions' => $permissions,
1938+
'field1' => 'value1',
1939+
'field2' => 'value2',
1940+
'field3' => 'value3',
1941+
'field4' => 'value4',
1942+
]));
1943+
1944+
// Upsert keeping only field1
1945+
$database->upsertDocument($collectionName, new Document([
1946+
'$id' => 'doc2',
1947+
'$permissions' => $permissions,
1948+
'field1' => 'updated_value1',
1949+
]));
1950+
1951+
$retrieved2 = $database->getDocument($collectionName, 'doc2');
1952+
$this->assertEquals('updated_value1', $retrieved2->getAttribute('field1'));
1953+
$this->assertArrayNotHasKey('field2', $retrieved2->getArrayCopy());
1954+
$this->assertArrayNotHasKey('field3', $retrieved2->getArrayCopy());
1955+
$this->assertArrayNotHasKey('field4', $retrieved2->getArrayCopy());
1956+
1957+
// Test 3: Remove nested object fields
1958+
$doc3 = $database->createDocument($collectionName, new Document([
1959+
'$id' => 'doc3',
1960+
'$permissions' => $permissions,
1961+
'name' => 'Product',
1962+
'details' => [
1963+
'color' => 'red',
1964+
'size' => 'large',
1965+
'weight' => 10
1966+
],
1967+
'specs' => [
1968+
'cpu' => 'Intel',
1969+
'ram' => '8GB'
1970+
]
1971+
]));
1972+
1973+
// Upsert removing details but keeping specs
1974+
$database->upsertDocument($collectionName, new Document([
1975+
'$id' => 'doc3',
1976+
'$permissions' => $permissions,
1977+
'name' => 'Updated Product',
1978+
'specs' => [
1979+
'cpu' => 'AMD',
1980+
'ram' => '16GB'
1981+
],
1982+
// details is removed
1983+
]));
1984+
1985+
$retrieved3 = $database->getDocument($collectionName, 'doc3');
1986+
$this->assertEquals('Updated Product', $retrieved3->getAttribute('name'));
1987+
$this->assertArrayHasKey('specs', $retrieved3->getArrayCopy());
1988+
$this->assertEquals('AMD', $retrieved3->getAttribute('specs')['cpu']);
1989+
$this->assertArrayNotHasKey('details', $retrieved3->getArrayCopy());
1990+
1991+
// Test 4: Remove array fields
1992+
$doc4 = $database->createDocument($collectionName, new Document([
1993+
'$id' => 'doc4',
1994+
'$permissions' => $permissions,
1995+
'title' => 'Article',
1996+
'tags' => ['tag1', 'tag2', 'tag3'],
1997+
'categories' => ['cat1', 'cat2'],
1998+
'comments' => ['comment1', 'comment2']
1999+
]));
2000+
2001+
// Upsert removing tags and comments but keeping categories
2002+
$database->upsertDocument($collectionName, new Document([
2003+
'$id' => 'doc4',
2004+
'$permissions' => $permissions,
2005+
'title' => 'Updated Article',
2006+
'categories' => ['cat3'],
2007+
]));
2008+
2009+
$retrieved4 = $database->getDocument($collectionName, 'doc4');
2010+
$this->assertEquals('Updated Article', $retrieved4->getAttribute('title'));
2011+
$this->assertArrayHasKey('categories', $retrieved4->getArrayCopy());
2012+
$this->assertEquals(['cat3'], $retrieved4->getAttribute('categories'));
2013+
$this->assertArrayNotHasKey('tags', $retrieved4->getArrayCopy());
2014+
$this->assertArrayNotHasKey('comments', $retrieved4->getArrayCopy());
2015+
2016+
// Test 5: upsertDocuments with field removal (bulk upsert)
2017+
$docs5 = [
2018+
new Document([
2019+
'$id' => 'bulk1',
2020+
'$permissions' => $permissions,
2021+
'fieldA' => 'valueA',
2022+
'fieldB' => 'valueB',
2023+
'fieldC' => 'valueC',
2024+
]),
2025+
new Document([
2026+
'$id' => 'bulk2',
2027+
'$permissions' => $permissions,
2028+
'fieldX' => 'valueX',
2029+
'fieldY' => 'valueY',
2030+
'fieldZ' => 'valueZ',
2031+
]),
2032+
];
2033+
$database->createDocuments($collectionName, $docs5);
2034+
2035+
// Upsert removing some fields from each
2036+
$upsertDocs5 = [
2037+
new Document([
2038+
'$id' => 'bulk1',
2039+
'$permissions' => $permissions,
2040+
'fieldA' => 'updatedA',
2041+
// fieldB and fieldC removed
2042+
]),
2043+
new Document([
2044+
'$id' => 'bulk2',
2045+
'$permissions' => $permissions,
2046+
'fieldX' => 'updatedX',
2047+
'fieldZ' => 'updatedZ',
2048+
// fieldY removed
2049+
]),
2050+
];
2051+
$database->upsertDocuments($collectionName, $upsertDocs5);
2052+
2053+
$retrievedBulk1 = $database->getDocument($collectionName, 'bulk1');
2054+
$this->assertEquals('updatedA', $retrievedBulk1->getAttribute('fieldA'));
2055+
$this->assertArrayNotHasKey('fieldB', $retrievedBulk1->getArrayCopy());
2056+
$this->assertArrayNotHasKey('fieldC', $retrievedBulk1->getArrayCopy());
2057+
2058+
$retrievedBulk2 = $database->getDocument($collectionName, 'bulk2');
2059+
$this->assertEquals('updatedX', $retrievedBulk2->getAttribute('fieldX'));
2060+
$this->assertEquals('updatedZ', $retrievedBulk2->getAttribute('fieldZ'));
2061+
$this->assertArrayNotHasKey('fieldY', $retrievedBulk2->getArrayCopy());
2062+
2063+
// Test 6: Upsert creating new document (should not unset anything)
2064+
$newDoc = $database->upsertDocument($collectionName, new Document([
2065+
'$id' => 'newDoc',
2066+
'$permissions' => $permissions,
2067+
'newField' => 'newValue',
2068+
]));
2069+
2070+
$this->assertEquals('newValue', $newDoc->getAttribute('newField'));
2071+
$retrievedNew = $database->getDocument($collectionName, 'newDoc');
2072+
$this->assertEquals('newValue', $retrievedNew->getAttribute('newField'));
2073+
$this->assertArrayHasKey('newField', $retrievedNew->getArrayCopy());
2074+
2075+
// Test 7: Remove all custom fields (keep only system fields)
2076+
$doc7 = $database->createDocument($collectionName, new Document([
2077+
'$id' => 'doc7',
2078+
'$permissions' => $permissions,
2079+
'custom1' => 'value1',
2080+
'custom2' => 'value2',
2081+
'custom3' => 'value3',
2082+
]));
2083+
2084+
// Upsert with only system fields (no custom fields)
2085+
$database->upsertDocument($collectionName, new Document([
2086+
'$id' => 'doc7',
2087+
'$permissions' => $permissions,
2088+
// No custom fields
2089+
]));
2090+
2091+
$retrieved7 = $database->getDocument($collectionName, 'doc7');
2092+
$this->assertArrayNotHasKey('custom1', $retrieved7->getArrayCopy());
2093+
$this->assertArrayNotHasKey('custom2', $retrieved7->getArrayCopy());
2094+
$this->assertArrayNotHasKey('custom3', $retrieved7->getArrayCopy());
2095+
// System fields should still exist
2096+
$this->assertEquals('doc7', $retrieved7->getId());
2097+
$this->assertNotNull($retrieved7->getCreatedAt());
2098+
$this->assertNotNull($retrieved7->getUpdatedAt());
2099+
2100+
// Test 8: Mixed scenario - add new fields while removing others
2101+
$doc8 = $database->createDocument($collectionName, new Document([
2102+
'$id' => 'doc8',
2103+
'$permissions' => $permissions,
2104+
'oldField1' => 'old1',
2105+
'oldField2' => 'old2',
2106+
'keepField' => 'keep',
2107+
]));
2108+
2109+
// Upsert removing oldField1 and oldField2, keeping keepField, adding newField
2110+
$database->upsertDocument($collectionName, new Document([
2111+
'$id' => 'doc8',
2112+
'$permissions' => $permissions,
2113+
'keepField' => 'updatedKeep',
2114+
'newField' => 'newValue',
2115+
]));
2116+
2117+
$retrieved8 = $database->getDocument($collectionName, 'doc8');
2118+
$this->assertEquals('updatedKeep', $retrieved8->getAttribute('keepField'));
2119+
$this->assertEquals('newValue', $retrieved8->getAttribute('newField'));
2120+
$this->assertArrayNotHasKey('oldField1', $retrieved8->getArrayCopy());
2121+
$this->assertArrayNotHasKey('oldField2', $retrieved8->getArrayCopy());
2122+
2123+
// Test 9: Verify internal/system fields are never removed
2124+
$doc9 = $database->createDocument($collectionName, new Document([
2125+
'$id' => 'doc9',
2126+
'$permissions' => $permissions,
2127+
'data' => 'test',
2128+
]));
2129+
2130+
$originalCreatedAt = $doc9->getCreatedAt();
2131+
$originalUpdatedAt = $doc9->getUpdatedAt();
2132+
2133+
// Upsert - internal fields should be preserved
2134+
$database->upsertDocument($collectionName, new Document([
2135+
'$id' => 'doc9',
2136+
'$permissions' => $permissions,
2137+
'newData' => 'newTest',
2138+
]));
2139+
2140+
$retrieved9 = $database->getDocument($collectionName, 'doc9');
2141+
// System fields should still exist
2142+
$this->assertEquals('doc9', $retrieved9->getId());
2143+
$this->assertEquals($originalCreatedAt, $retrieved9->getCreatedAt());
2144+
// UpdatedAt should be different (document was updated)
2145+
$this->assertNotEquals($originalUpdatedAt, $retrieved9->getUpdatedAt());
2146+
$this->assertEquals('newTest', $retrieved9->getAttribute('newData'));
2147+
// Old field should be removed
2148+
$this->assertArrayNotHasKey('data', $retrieved9->getArrayCopy());
2149+
2150+
// Clean up
2151+
$database->deleteCollection($collectionName);
2152+
}
18682153
}

0 commit comments

Comments
 (0)