mirror of
https://github.com/digital-asset/daml.git
synced 2024-09-20 01:07:18 +03:00
[DPP-1327] Efficient Oracle ETQ data migration (#15843)
Previous version of etq_array_diff function proved to be vastly inefficient on a synthetic data set resulting in "SQL Error: ORA-04036: PGA memory used by the instance exceeds PGA_AGGREGATE_LIMIT" error after many minutes of processing. PGA_AGGREGATE_LIMIT was set to 2GB in that testing setup. This new implementation of etq_array_diff completed the migration successfully in 03min 34s which is comparable with the data migration for PostgreSQL. No evidence for excessive consumption of memory was found ('select * from V$PGASTAT;').
This commit is contained in:
parent
e880f26b36
commit
fe92e52da7
@ -1 +1 @@
|
||||
871a62483f3ea549e765e3b199eb0a9c116b2a6d6408918dd101aaf6d5899059
|
||||
b58ff4ac8a8a2be798424de9dd246dedb1825bfe468f00fc82ba65c534654af7
|
||||
|
@ -2,30 +2,26 @@
|
||||
|
||||
-- Removes all elements from a that are present in b, essentially computes a - b.
|
||||
CREATE OR REPLACE FUNCTION etq_array_diff(
|
||||
arrayClob1 IN CLOB,
|
||||
arrayClob2 IN CLOB
|
||||
clobA IN CLOB,
|
||||
clobB IN CLOB
|
||||
)
|
||||
RETURN CLOB
|
||||
IS
|
||||
arrayJson1 json_array_t := json_array_t.parse(arrayClob1);
|
||||
outputJsonArray json_array_t := json_array_t ('[]');
|
||||
-- Number type has
|
||||
-- 999...(38 9's) x10^125 maximum value
|
||||
-- -999...(38 9's) x10^125 minimum value
|
||||
-- so 200 characters should be enough to hold it together with the whole filter expression
|
||||
filterExpression varchar2(200);
|
||||
aDiffB CLOB;
|
||||
BEGIN
|
||||
FOR i IN 0 .. arrayJson1.get_size - 1
|
||||
LOOP
|
||||
-- `$[*]` selects each element of the array
|
||||
-- `(@ == v)` is a filter expression that check whether each matched element is equal to some value `v`
|
||||
filterExpression := '$[*]?(@ == ' || (arrayJson1.get(i).to_clob()) ||')';
|
||||
IF NOT json_exists(arrayClob2, filterExpression)
|
||||
THEN
|
||||
outputJsonArray.append(arrayJson1.get(i));
|
||||
END IF;
|
||||
END LOOP;
|
||||
RETURN outputJsonArray.to_clob();
|
||||
SELECT coalesce(JSON_ARRAYAGG(elemA), '[]') foo
|
||||
INTO aDiffB
|
||||
FROM
|
||||
(
|
||||
SELECT elemA FROM json_table(clobA, '$[*]' columns (elemA NUMBER PATH '$'))
|
||||
) arrayA
|
||||
LEFT JOIN
|
||||
(
|
||||
SELECT elemB FROM json_table(clobB, '$[*]' columns (elemB NUMBER PATH '$'))
|
||||
) arrayB
|
||||
ON elemA = elemB
|
||||
WHERE elemB IS NULL;
|
||||
RETURN aDiffB;
|
||||
END;
|
||||
/
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user