run_compress.sh
author Peter Gervai <grin@grin.hu>
Wed, 24 Mar 2021 10:16:42 +0100
changeset 0 4216bf35912e
permissions -rwxr-xr-x
Init repo; my matrix stuff * add media_purge * add run_compress * filtered _config.sh
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
0
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
     1
#!/bin/sh
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
     2
# compress synapse database room graphs
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
     3
# (cc) Peter 'grin' Gervai, 2020
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
     4
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
     5
ARG="$1"
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
     6
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
     7
source _config.sh
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
     8
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
     9
set -x
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    10
CMD="SELECT room_id, COUNT(*), type FROM state_groups_state GROUP BY 1,3 HAVING COUNT(*)>999999 ORDER BY 2 DESC;"
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    11
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    12
BASE="/root/matrix-synapse/clean_auto"
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    13
LOG="${BASE}/process.log"
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    14
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    15
savelog -c 4 $LOG
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    16
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    17
TMP=`mktemp /tmp/synapse_compress.XXXXXXXX`
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    18
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    19
echo "==== `date +%F_%T` now." 						>> ${LOG}
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    20
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    21
if [ "x$ARG" = "x" ]; then
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    22
	echo "Collecting room/state groups data..."				| tee -a ${LOG}
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    23
	if ! psql "${DB}" synapse -tAc "${CMD}" > $TMP; then
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    24
		echo "Error getting room stats. Exit."
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    25
		exit 1
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    26
	fi
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    27
else
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    28
	echo "Processing already collected data ${ARG}."
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    29
	rm "${TMP}"
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    30
	TMP="$ARG"
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    31
fi
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    32
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    33
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    34
awk -F'|' '{print $1}' < $TMP | while read room; do
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    35
	NOW=`date +%F_%T`
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    36
	echo "Processing room ${room}... into c${NOW}_compress.sql" 	| tee -a ${LOG}
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    37
	/usr/bin/time ../synapse-compress-state -o "c${NOW}_compress.sql" -p "${DB}" -t -r "${room}" 	| tee -a ${LOG}
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    38
	echo "Actually compressing (c${NOW}_compress.sql)..."		| tee -a ${LOG}
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    39
	/usr/bin/time psql "${DBRW}" synapse -f "c${NOW}_compress.sql" 	| tee -a ${LOG}
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    40
done
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    41
echo "We shall delete ${TMP} file."					| tee -a ${LOG}
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    42
set +x
4216bf35912e Init repo; my matrix stuff
Peter Gervai <grin@grin.hu>
parents:
diff changeset
    43