diff -r 000000000000 -r 3b714bbb1347 delinquent_files/demon.php
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/delinquent_files/demon.php	Mon Jan 23 21:37:02 2023 +0100
@@ -0,0 +1,706 @@
+#!/usr/bin/php
+<?PHP
+
+chdir ( '/data/project/commons-delinquent' ) ;
+
+error_reporting(E_ERROR|E_CORE_ERROR|E_COMPILE_ERROR); # E_ALL|
+ini_set('display_errors', 'On');
+
+require_once ( './shared.inc' ) ;
+require_once( './matcher.inc' );
+require_once( './debug.inc' );
+
+class CommonsDelinquentDemon extends CommonsDelinquent {
+
+	var $avoidNamespaceOnWiki = [
+		'dewiki' => [4]
+	] ;
+
+	private $d;
+	private $matcher;
+
+	var $maximum_file_usage_limit = 65535; # prevent memory explosion by extreme used files (65535 ~ 650 MB)
+	var $delay_minutes = 10 ;  # Wait after deletion
+	var $fallback_minutes = 120 ; # Only used if DB is empty
+	var $max_text_diff = 1500 ; # Max char diff
+	var $min_faux_template_icon = 500 ;
+	var $comments = array() ;
+	var $comments_default = array (
+		'summary' => 'Removing [[:c:File:$1|$1]], it has been deleted from Commons by [[:c:User:$2|$2]] because: $3.' ,
+		'replace' => 'Replacing $1 with [[File:$2]] (by [[:c:User:$3|$3]] because: $4).' ,
+		'by' => ' Requested by [[User:$1|]].'
+	) ;
+
+	function __construct() {
+		parent::__construct();
+		$this->d = new Debug;
+		$this->matcher = new Matcher($this->d);
+	}
+
+	function set_debug_mode($l) {
+		$this->d->set_level( $l );
+	}
+
+	function debug($msg) {
+		$this->d->debug($msg);
+	}
+
+	// Returns the last timestamp in the tool database, or a dummy (current time - X min)
+	function getLastTimestamp () {
+		# Open tool database
+		$db = $this->getToolDB() ;
+		
+		# Get highest timestamp in tool DB as a starting point
+		$max_ts = '' ;
+		$sql = "SELECT max(log_timestamp) AS max_ts FROM event WHERE done=1" ; # Timestamp of Commons logging table, NOT tool edit timestamp!
+		$result = $this->runQuery ( $db , $sql ) ;
+		while($o = $result->fetch_object()){
+			$max_ts = $o->max_ts ;
+		}
+		$db->close() ;
+		if ( $max_ts == '' ) $max_ts = date ( 'YmdHis' , time() - $this->fallback_minutes*60 ) ; # Fallback to current date minus X min
+		return $max_ts ;
+	}
+	
+	function isBadPage ( $o , $filename ) {
+		if ( $o->gil_page_namespace_id == 6 and $o->gil_wiki == 'commonswiki' and $o->gil_to == $filename ) return true ; // Self-reference
+		if ( $o->gil_page_namespace_id == 2 and $o->gil_wiki == 'commonswiki' and preg_match ( '/^\w+Bot\b/' , $o->gil_page_title ) ) return true ; // Bot subpage on Commons
+		if ( $o->gil_page_namespace_id == 4 and $o->gil_wiki == 'commonswiki' and preg_match ( '/(Deletion(_| )requests\/.*|Undeletion(_| )requests\/.*)\b/' , $o->gil_page_title ) ) return true ; // DR and UDR on Commons
+		foreach ( $this->avoidNamespaceOnWiki AS $wiki => $namespaces ) {
+			if ( $o->gil_wiki != $wiki ) continue ;
+			foreach ( $namespaces AS $namespace ) {
+				if ( $namespace == $o->gil_page_namespace_id ) return true ;
+			}
+		}
+		return false ;
+	}
+	
+	function getRecentDeletedFiles ( $max_ts ) {
+		# Open Commons database replica
+		$db_co = $this->getCommonsDB() ;
+		$cur_ts = date ( 'YmdHis' , time() - $this->delay_minutes*60 ) ;
+
+		# Get all file deletions
+		$delink_files = array() ; # Files to delink
+		$sql = "SELECT * FROM logging_compat LEFT JOIN comment ON comment_id = log_comment_id WHERE log_type='delete' AND log_action='delete' AND log_timestamp>='$max_ts' AND log_timestamp<'$cur_ts' AND log_namespace=6" ;
+		$sql .= " AND NOT EXISTS (SELECT * FROM image WHERE img_name=log_title)" ;
+		$sql .= " AND NOT EXISTS (SELECT * FROM page WHERE page_title=log_title AND page_namespace=6 AND page_is_redirect=1)" ; # Do not remove redirects. Is that OK???
+		$sql .= " ORDER BY log_timestamp ASC" ;
+		$result = $this->runQuery ( $db_co , $sql ) ;
+		while($o = $result->fetch_object()){
+			$delink_files[] = $o ;
+		}
+		foreach ( $delink_files AS $deletion ) {
+			$filename = $deletion->log_title ;
+			$sql = "SELECT * FROM globalimagelinks WHERE gil_to='" . $this->getDBsafe($filename) . "'" ;
+			$deletion->usage = array() ; # Usage instances for this file
+			$result = $this->runQuery ( $db_co , $sql ) ;
+			while($o = $result->fetch_object()){
+				if ( $this->isBadWiki($o->gil_wiki) ) continue ;
+				if ( $this->isBadPage($o,$filename) ) continue ;
+				$deletion->usage[] = $o ;
+			}
+		}
+		$db_co->close() ;
+//		print_r ( $delink_files ) ;
+		return $delink_files ;
+	}
+
+	function getFileUsage ( $filename ) {
+		$this->d->trace("Get file usage for $filename");
+		$ret = array() ;
+		$db_co = $this->getCommonsDB() ;
+		$cur_ts = date ( 'YmdHis' , time() - $this->delay_minutes*60 ) ;
+		$sql = "SELECT * FROM globalimagelinks WHERE gil_to='" . $this->getDBsafe($filename) . "'" ;
+		$this->d->trace("Try running: $sql");
+		$result = $this->runQuery ( $db_co , $sql ) ;
+		$this->d->trace("Got result, looping through");
+		while($o = $result->fetch_object()){
+			if ( $this->isBadWiki($o->gil_wiki) ) continue ;
+			$ret[] = $o ;
+			// $this->d->trace("Added to ret, len=" . count($ret));
+			# limit the maximum matches because we use more memory than toolforge allows
+			if( count($ret) > $this->maximum_file_usage_limit ) {
+				$this->d->error("Overflow!! We cannot get usage for $filename, too much hits (>$this->maximum_file_usage_limit)!");
+				$ret = array();
+				$ret[] = "*OVERFLOW*";
+				$db_co->close();
+				return $ret;
+			}
+		}
+		$this->d->trace("Processed " . count($ret) . " usage points");
+		$db_co->close() ;
+		return $ret ;
+	}
+	
+	function canUnlinkFromNamespace ( $usage ) {
+		if ( $usage->gil_page_namespace_id % 2 > 0 ) return false ; // Skip talk pages
+		if ( $usage->gil_page_namespace_id < 0 ) return false ; // Paranoia
+		return true ;
+	}
+
+	function fileExistenceSanityCheck ( $e , $check_commons ) {
+		if ( $this->hasLocalFile ( $e->wiki , $e->file ) ) {
+			$this->setDone ( $e->id , 2 , 'Skipped: Local file exists' ) ;
+			return false ;
+		}
+		if ( $check_commons and $this->hasLocalFile ( 'commonswiki' , $e->file ) ) {
+			$this->setDone ( $e->id , 2 , 'Skipped: Commons file exists' ) ;
+			return false ;
+		}
+		return true ;
+	}
+
+
+	function getTextFromWiki ( $wiki , $pagename ) {
+		$ret = false ;
+		$api = $this->getAPI ( $wiki ) ;
+		if ( $api ) {
+			$services = new \Mediawiki\Api\MediawikiFactory( $api );
+			$page = $services->newPageGetter()->getFromTitle( $pagename );
+			$revision = $page->getRevisions()->getLatest();
+		
+			if ( $revision ) {
+				$ret = $revision->getContent()->getData() ;
+			}
+		}
+		return $ret ;
+	}
+	
+	/**
+		mode	"summary" or "replace"
+	*/
+	function getLocalizedCommentPattern ( $wiki , $mode = 'summary') {
+		if ( !isset($mode) ) $mode = 'summary' ;
+		if ( isset ( $this->comments[$mode][$wiki] ) ) return $this->comments[$mode][$wiki] ;
+		$pattern = $this->comments_default[$mode] ; # Default
+		
+		# Try local translation
+		$local = $this->getTextFromWiki ( $wiki , 'User:CommonsDelinker/' . $mode . '-I18n' ) ;
+		if ( $local !== false ) $pattern = $local ;
+		
+		$this->comments[$mode][$wiki] = $pattern ;
+		return $pattern ;
+	}
+	
+	function constructUnlinkComment ( $file , $usage ) {
+		$pattern = $this->getLocalizedCommentPattern ( $usage->gil_wiki ) ;
+		
+		$c = $file->comment_text ;
+		if ( $usage->wiki != 'commonswiki' ) { # Point original comment links to Commons
+			$c = preg_replace ( '/\[\[([^|]+?)\]\]/' , '[[:c:\1|]]' , $c ) ; # Pointing to Commons (no pipe)
+			$c = preg_replace ( '/\[\[([^:].+?)\]\]/' , '[[:c:\1]]' , $c ) ; # Pointing to Commons (with pipe)
+		}
+
+		$pattern = preg_replace ( '/\$1/' , $file->log_title , $pattern ) ;
+		$pattern = preg_replace ( '/\$2/' , $file->log_user_text , $pattern ) ;
+		$pattern = preg_replace ( '/\$3/' , $c , $pattern ) ;
+#		print "\n$pattern\n" ; exit ( 0 ) ; // TESTING
+		return $pattern ;
+	}
+
+	function constructReplaceComment ( $params ) {
+		$pattern = $this->getLocalizedCommentPattern ( $params['wiki'] , 'replace' ) ;
+		
+		$c = $params['comment'] ;
+		if ( $params['wiki'] != 'commonswiki' ) { # Point original comment links to Commons
+			$c = preg_replace ( '/\[\[([^|]+?)\]\]/' , '[[:c:\1|]]' , $c ) ; # Pointing to Commons (no pipe)
+			$c = preg_replace ( '/\[\[([^:].+?)\]\]/' , '[[:c:\1]]' , $c ) ; # Pointing to Commons (with pipe)
+		}
+
+		$pattern = preg_replace ( '/\$1/' , $params['file'] , $pattern ) ;
+		$pattern = preg_replace ( '/\$2/' , $params['replace_with_file'] , $pattern ) ;
+		$pattern = preg_replace ( '/\$3/' , 'CommonsDelinker' , $pattern ) ;
+		$pattern = preg_replace ( '/\$4/' , $c , $pattern ) ;
+		
+		if ( isset($params['user']) and $params['user'] != '' ) {
+			$by = $this->getLocalizedCommentPattern ( $params['wiki'] , 'by' ) ;
+			$by = preg_replace ( '/\$1/' , $params['user'] , $by ) ;
+			$pattern .= ' ' . $by ;
+		}
+		
+		return $pattern ;
+	}
+	
+	function addUnlinkEvent ( $file , $usage , &$sqls ) {
+		if ( !$this->canUnlinkFromNamespace ( $usage ) ) return ;
+		if ( $this->hasLocalFile ( $usage->gil_wiki , $usage->gil_to ) ) return ;
+		
+		$page = $usage->gil_page_title ;
+		if ( $usage->gil_page_namespace != '' ) $page = $usage->gil_page_namespace . ":$page" ;
+		$params = array (
+			'action' => 'unlink' ,
+			'file' => $usage->gil_to ,
+			'wiki' => $usage->gil_wiki ,
+			'page' => $page ,
+			'namespace' => $usage->gil_page_namespace_id ,
+			'comment' => $this->constructUnlinkComment ( $file , $usage ) ,
+			'timestamp' => date ( 'YmdHis' ) ,
+			'log_id' => $file->log_id ,
+			'log_timestamp' => $file->log_timestamp ,
+			'done' => 0
+		) ;
+#		print_r ( $params ) ;
+		
+		$s1 = array() ;
+		$s2 = array() ;
+		foreach ( $params AS $k => $v ) {
+			$s1[] = $k ;
+			$s2[] = "'" . $this->getDBsafe($v) . "'" ;
+		}
+		
+		$sql = "INSERT IGNORE INTO event (" . implode ( ',' , $s1 ) . ") VALUES (" . implode ( "," , $s2 ) . ")" ;
+		$sqls[] = $sql ;
+	}
+	
+	function addUnlinkEvents ( $delink_files ) {
+		$sqls = array() ;
+		foreach ( $delink_files AS $file ) {
+			foreach ( $file->usage AS $usage ) {
+				$this->addUnlinkEvent ( $file , $usage , $sqls ) ;
+			}
+		}
+		
+		$db = $this->getToolDB() ;
+		foreach ( $sqls AS $sql ) $this->runQuery ( $db , $sql ) ;
+		$db->close() ;
+	}
+	
+	function getJSON4Q ( $e ) {
+		$q = $e->page ;
+		$url = "http://www.wikidata.org/w/api.php?action=wbgetentities&format=json&ids=" . $q ;
+		$j = json_decode ( file_get_contents ( $url ) ) ;
+		if ( isset ( $j->entities->$q->missing ) ) { # No such item
+			$this->setDone ( $e->id , 2 , "No such item $q" ) ;
+			return false ;
+		}
+		if ( !isset ( $j->entities->$q->claims ) ) {
+			$this->setDone ( $e->id , 2 , "Did not find " . $e->file . " on " . $q ) ;
+			return false ;
+		}
+		return $j ;
+	}
+
+	function performEditUnlinkWikidata ( $e ) {
+		$j = $this->getJSON4Q ( $e ) ;
+		if ( $j === false ) return ;
+
+		$q = $e->page ;
+		$j = $j->entities->$q->claims ;
+		$remove = array() ;
+		foreach ( $j AS $prop => $claims ) {
+			foreach ( $claims AS $c ) {
+				if ( $c->type != 'statement' ) continue ;
+				if ( $c->mainsnak->datatype != 'commonsMedia' ) continue ;
+				if ( str_replace ( ' ' , '_' , ucfirst ( trim ( $c->mainsnak->datavalue->value ) ) ) != $e->file ) continue ;
+				$remove[] = $c->id ;
+			}
+		}
+		
+		if ( count($remove) > 0 ) {
+			$ok = $this->editWikidata ( 'wbremoveclaims' , array ( 'claim'=>implode('|',$remove) , 'summary' => $e->comment ) ) ;
+			if ( !$ok ) return ;
+		}
+		
+		$this->setDone ( $e->id , 1 ) ; # OK!
+	}
+
+
+	function performEditReplaceWikidata ( $e ) {
+		$j = $this->getJSON4Q ( $e ) ;
+		if ( $j === false ) return ;
+
+		$q = $e->page ;
+
+		$j = $j->entities->$q->claims ;
+		$remove = array() ;
+		foreach ( $j AS $prop => $claims ) {
+			foreach ( $claims AS $c ) {
+				if ( $c->type != 'statement' ) continue ;
+				if ( $c->mainsnak->datatype != 'commonsMedia' ) continue ;
+				if ( str_replace ( ' ' , '_' , ucfirst ( trim ( $c->mainsnak->datavalue->value ) ) ) != $e->file ) continue ;
+				$remove[] = array ( $c->id , $prop ) ;
+			}
+		}
+		
+		if ( count($remove) > 0 ) {
+
+			# Remove old image entries
+			$ids = array() ;
+			foreach ( $remove AS $r ) $ids[] = $r[0] ;
+			$ok = $this->editWikidata ( 'wbremoveclaims' , array ( 'claim'=>implode('|',$ids) ) ) ;
+			if ( !$ok ) {
+				$this->d->error("performEditReplaceWikidata:1 failed");
+				return ;
+			}
+
+			# Add new image entries
+			foreach ( $remove AS $r ) {
+				$params = array(
+					'snaktype' => 'value' ,
+					'property' => $r[1] ,
+					'value' => json_encode(str_replace('_',' ',$e->replace_with_file)) ,
+					'entity' => $e->page ,
+					'summary' => $e->comment
+				) ;
+
+				$ok = $this->editWikidata ( 'wbcreateclaim' , $params ) ;
+				if ( !$ok ) {
+					$this->d->error( "performEditReplaceWikidata:2 failed" );
+					return ;
+				}
+
+			}
+		} else {
+			$this->setDone ( $e->id , 2 , 'File link not found in page' ) ;
+			return ;
+		}
+		
+		$this->setDone ( $e->id , 1 ) ; # OK!
+	}
+
+	
+	##################################################################
+	##
+	## Here we make the changes.
+	## Get the page, replace content and upload again.
+	##
+	##################################################################
+	function performEditText ( $e ) {
+		$this->d->debug( "performEditText $e->action on id=$e->id wiki=$e->wiki page=$e->page." );
+		$api = $this->getAPI ( $e->wiki ) ;
+		if ( $api === false ) {
+			$this->setDone ( $e->id , 2 , "Could not connect to API" ) ;
+			return ;
+		}
+		$services = new \Mediawiki\Api\MediawikiFactory( $api );
+		try {
+			$page = $services->newPageGetter()->getFromTitle( $e->page );
+		} catch (Exception $ex) {
+			$this->setDone ( $e->id , 2 , "Page not found" ) ;
+			$this->d->debug("Page '$e->page' not found ($ex), id='$e->id' wiki=$e->wiki file=$e->file action=$e->action");
+			return ;
+		}
+		$revision = $page->getRevisions()->getLatest();
+		
+		if ( !$revision ) {
+			$this->setDone ( $e->id , 2 , "Latest revision not found" ) ;
+			return ;
+		}
+		
+		$rev_id = $revision->getId() ;
+		$text = $revision->getContent()->getData() ;
+		
+		$file = $e->file ;
+		$pattern = $this->matcher->matcher_prepare_pattern( $file );
+	
+		$new_text = $text ;
+
+		if ( $e->action == 'unlink' ) {
+			## remove image references in text; see ./matcher.inc
+			$this->d->info("Try to unlink '$pattern' in $e->wiki: $e->page");
+			$new_text = $this->matcher->matcher_do_unlink( $new_text, $pattern );
+
+		} else if ( $e->action == 'replace' ) {
+			## replace image with new_file in text; see ./matcher.inc
+			$new_file = $e->replace_with_file;
+			$this->d->info("Try to replace '$pattern' with '$new_file' in $e->wiki: $e->page");
+			$new_text = $this->matcher->matcher_do_replacement( $new_text, $pattern, $new_file );
+		}
+		
+		if ( $text == $new_text ) { # No change
+			$this->setDone ( $e->id , 2 , 'File link not found in page' ) ;
+			$this->d->info( "Article unchanged. id=$e->id; $e->wiki: $e->page" );
+			return ;
+		}
+		
+		if ( strlen(trim($new_text)) == 0 or abs(strlen($text)-strlen($new_text)) > $this->max_text_diff ) {
+			$this->setDone ( $e->id , 2 , 'Text change too big' ) ;
+			$this->d->warn( "Article text change is too big. id=$e->id; $e->wiki: $e->page" );
+			return ;
+		}
+		
+		if ( !isset($e->comment) ) $e->comment = '' ;
+		$e->comment = (string)$e->comment ;
+
+		$this->d->info( "Editing $e->wiki: $e->page to $e->action $e->file (w/ $new_file) AS $e->comment") ;
+		
+		$params = array (
+			'title' => $e->page ,
+			'text' => trim($new_text) ,
+			'summary' => $e->comment ,
+			'bot' => 1
+		) ;
+		
+		$x = $this->editWiki ( $e->wiki , 'edit' , $params ) ;
+		if ( $x and $x['edit']['result'] == 'Success' ) {
+			$this->setDone ( $e->id , 1 , array('revision'=>$rev_id) ) ;
+		} else {
+			$this->d->error( "Cannot edit wiki ($e->wiki: $e->page): " . $this->last_exception );
+			$this->setDone ( $e->id , 2 , $this->last_exception ) ;
+		}
+
+	}
+	
+	function performEditReplace ( $e ) {
+		if ( !$this->fileExistenceSanityCheck($e,false) ) return ; # Nothing to do
+		if ( !isset($e->namespace) ) return ; # Paranoia
+		if ( $e->wiki == 'wikidatawiki' && $e->namespace == 0 ) { # Wikidata item
+			$this->performEditReplaceWikidata ( $e ) ;
+		} else { # "Normal" edit
+			$this->performEditText ( $e ) ;
+		}
+	}
+	
+	function performEditUnlink ( $e ) {
+		if ( !$this->fileExistenceSanityCheck($e,true) ) return ; # Nothing to do
+		if ( $e->wiki == 'wikidatawiki' && $e->namespace == 0 ) { # Wikidata item
+			$this->performEditUnlinkWikidata ( $e ) ;
+		} else { # "Normal" edit
+			$this->d->debug( "performEditUnlink $e->action on id=$e->id wiki=$e->wiki page=$e->page." );
+			$this->performEditText ( $e ) ;
+		}
+	}
+	
+	function performEdit ( $e ) {
+		$this->d->debug( "performEdit $e->action on id=$e->id wiki=$e->wiki page=$e->page." );
+		if ( $e->action == 'unlink' ) $this->performEditUnlink ( $e ) ;
+		else if ( $e->action == 'replace' ) $this->performEditReplace ( $e ) ;
+		else {
+			$this->d->error( "PerformEdit got unknown action $e->action" );
+			print_r ( $e ) ;
+			die ( "Unknown action " . $e->action ) ;
+		}
+	}
+	
+	function clearBogusIssues ( $db ) {
+		# Clear some previous issues
+		// 0=pending
+		// 1=done
+		// 2=skipped
+		$sql = "update `event` set done=0,note='' where note like '%rate limit%' and done=2" ;
+		$this->d->debug("Set done=0 (pending) on 'rate limit' events where done=2(skipped)");
+		$this->runQuery ( $db , $sql ) ;
+		$sql = "update `event` set done=0,note='' where note like '%edit conflict%' and done=2" ;
+		$this->d->debug("Set done=0 (pending) on 'edit conflict' events where done=2(skipped)");
+		$this->runQuery ( $db , $sql ) ;
+	}
+	
+	function performEdits () {
+		$edits = array() ;
+		$this->d->debug("Connecting to DB");
+		$db = $this->getToolDB() ;
+		## this is slow, let's do it at the end
+		// $this->d->debug("Clear bogus issues (reactivate rate limit/edit conflict skipped issues)");
+		// $this->clearBogusIssues ( $db ) ;
+		$this->d->debug("Get work events (pending events)");
+		$sql = "SELECT * FROM `event` WHERE done=0 ORDER BY timestamp ASC,log_timestamp ASC" ;
+		$result = $this->runQuery ( $db , $sql ) ;
+		while($o = $result->fetch_object()){
+			$edits[] = $o ;
+		}
+		$db->close() ;
+
+		$last_wiki = '' ;
+		foreach ( $edits AS $o ) {
+			if ( $last_wiki == $o->wiki ) sleep ( 5 ) ; // Edit rate limiter
+			$this->d->debug("Perform an edit in $o->wiki");
+			try {
+				$this->performEdit ( $o ) ;
+			} catch (Exception $e) {
+    			echo 'Caught exception: ',  $e->getMessage(), "\n";
+			}
+			$last_wiki = $o->wiki ;
+		}
+
+		$this->d->debug("Connecting to DB");
+		$db = $this->getToolDB() ;
+		$this->d->debug("Clear bogus issues (reactivate rate limit/edit conflict skipped issues)");
+		$this->clearBogusIssues ( $db ) ;
+		$db->close() ;
+	}
+	
+	function addReplaceEvents () {
+		$cmd_page = 'User:CommonsDelinker/commands' ;
+		$this->d->trace("getText from User:CommonsDelinker/commands");
+		$t = $this->getTextFromWiki ( 'commonswiki' , $cmd_page ) ;
+		if ( $t === false ) {
+			$this->d->error( "Could not open commands page") ;
+			return ;
+		}
+		
+		if ( preg_match ( '/\{\{[Ss]top\}\}/' , $t ) ) return ; // STOP
+		
+		$sqls = array() ;
+		
+#		$t = "{{/front}}\n{{universal replace|Overzicht - Hulst - 20118655 - RCE.jpg|Red Weaver Ant, Oecophylla smaragdina.jpg|reason=Testing}}" ; # TESTING
+		
+		$this->d->trace("Processing page content...");
+		$ts = date ( 'YmdHis' ) ;
+		$t = explode ( "\n" , $t ) ;
+		$nt = array() ;
+		foreach ( $t AS $l ) {
+			if ( !preg_match ( '/^\s*\{\{\s*[Uu]niversal[ _]replace\s*\|\s*(.+?)\s*\|\s*(.+?)\s*\|\s*reason\s*=\s*(\S.*?)\s*\}\}/' , $l , $m ) ) {
+				if ( !preg_match ( '/^\s*\{\{\s*[Uu]niversal[ _]replace\s*\|\s*(.+?)\s*\|\s*(.+?)\s*\|\s*reason\s*=\s*(.+?)\s*\|\s*user\s*=\s*(\S.*?)\s*\}\}/' , $l , $m ) ) {
+					$nt[] = $l ;
+					continue ;
+				}
+			}
+			$old_file = ucfirst(str_replace(' ','_',trim($m[1]))) ;
+			$new_file = ucfirst(str_replace(' ','_',trim($m[2]))) ;
+			$this->d->trace("Process line; old:$old_file new:$new_file.");
+			
+			$comment = trim($m[3]) ;
+			$user = '' ;
+			if ( isset($m[4]) ) {
+				$user = str_replace(' ','_',trim($m[4])) ;
+				$user = preg_replace ( '/^\s*\[\[[^:]+(.+?)\s*(\||\]\]).*$/' , '$1' , $user ) ;
+			}
+			
+			if ( !$this->hasLocalFile ( 'commonswiki' , $new_file ) ) {
+				$nt[] = "No such replacement file: " . $l ;
+				continue ;
+			}
+
+			if ( !preg_match('/\.svg$/i',$old_file) and preg_match('/\.svg$/i',$new_file) ) {
+				$nt[] = "Non-SVG to SVG replacement: " . $l ;
+				continue ;
+			}
+
+			$this->d->trace("Get file usage for $old_file");
+			$usages = $this->getFileUsage ( $old_file ) ;
+			if( $usages[0] == "*OVERFLOW*" ) {
+				$nt[] = "File is used on too many pages: " . $l;
+				$this->d->info("Skipping file $old_file; high usage (>$this->maximum_file_usage_limit)!");
+				continue;
+			}
+			
+			$this->d->trace("Get db handle for TooDB");
+			$db = $this->getToolDB() ;
+			
+			$this->d->trace("Generate replaces...");
+			foreach ( $usages AS $usage ) {
+				$page = $usage->gil_page_title ;
+				if ( $usage->gil_page_namespace != '' ) $page = $usage->gil_page_namespace . ':' . $page ;
+				$params = array (
+					'action' => 'replace' ,
+					'file' => $old_file ,
+					'wiki' => $usage->gil_wiki ,
+					'page' => $page ,
+					'namespace' => $usage->gil_page_namespace_id ,
+					'timestamp' => $ts ,
+					'comment' => $comment ,
+					'log_id' => -1 ,
+					'log_timestamp' => $ts ,
+					'user' => $user ,
+					'done' => 0 ,
+					'replace_with_file' => $new_file
+				) ;
+				$params['comment'] = $this->constructReplaceComment ( $params ) ;
+//				print_r ( $params ) ;
+
+				$s1 = array() ;
+				$s2 = array() ;
+				foreach ( $params AS $k => $v ) {
+					$s1[] = $k ;
+					$s2[] = "'" . $this->getDBsafe($v) . "'" ;
+				}
+				
+				$this->d->trace("Add SQL to sqls[] array, len=" . count($sqls));
+				$sql = "INSERT IGNORE INTO event (" . implode ( ',' , $s1 ) . ") VALUES (" . implode ( "," , $s2 ) . ")" ;
+				$sqls[] = $sql ;
+				$this->d->trace("SQL:$sql");
+
+			}
+			
+			$db->close() ;
+			
+		}
+		
+		$t = implode ( "\n" , $t ) ;
+		$nt = implode ( "\n" , $nt ) ;
+		if ( $t == $nt ) return ; // No change
+		
+		# Run SQL
+		$db = $this->getToolDB() ;
+		foreach ( $sqls AS $sql ) $this->runQuery ( $db , $sql ) ;
+		$db->close() ;
+		
+		# Save new text to Wiki
+		$params = array (
+			'title' => $cmd_page ,
+			'text' => trim($nt) ,
+			'summary' => 'Removing replace commands, will be executed soon' ,
+			'bot' => 1
+		) ;
+		
+		$this->d->info( "Editing $cmd_page...") ;
+		$x = $this->editWiki ( 'commonswiki' , 'edit' , $params ) ;
+		$this->d->debug( "Editing $cmd_page done.") ;
+	}
+	
+	function fixFauxTemplateReplacements () {
+		$todo = array() ;
+		$db = $this->getToolDB() ;
+		$sql = "DELETE FROM event WHERE action='' and file=''" ;
+		$result = $this->runQuery ( $db , $sql ) ;
+		$sql = 'select file,wiki, count(*) as cnt,namespace from event where done=0 group by file,wiki,namespace having cnt>' . $this->min_faux_template_icon ;
+		$result = $this->runQuery ( $db , $sql ) ;
+		while($o = $result->fetch_object()){
+			$file = $this->getDBsafe ( $o->file ) ;
+			$wiki = $this->getDBsafe ( $o->wiki ) ;
+			$todo[] = "UPDATE event SET done=2,note='Likely template icon, skipping' WHERE file='$file' AND wiki='$wiki' AND namespace=" . $o->namespace ;
+		}
+		foreach ( $todo AS $sql ) {
+			$this->runQuery ( $db , $sql ) ;
+		}
+		$db->close() ;
+	}
+
+	// Unlinks deleted files
+	function run () {
+		$this->d->debug("Get last timestamp");
+		$max_ts = $this->getLastTimestamp() ;
+		$this->d->debug("Get recent deleted files");
+		$delink_files = $this->getRecentDeletedFiles ( $max_ts ) ;
+		$this->d->debug("Add unlink events for recently deleted files");
+		$this->addUnlinkEvents ( $delink_files ) ;
+		$this->d->debug("Add replace events");
+		$this->addReplaceEvents () ;
+		$this->d->debug("Fix bogus template replacements");
+		$this->fixFauxTemplateReplacements() ;
+		$this->d->debug("Perform the queued edits");
+		$this->performEdits() ;
+	}
+	
+	function debug_run0() {
+		$this->d->debug("Add replace events");
+		$this->addReplaceEvents () ;
+	
+	}
+
+}
+
+print "Bot is starting.\n";
+$demon = new CommonsDelinquentDemon ;
+
+//$demon->addReplaceEvents () ;
+//$demon->performEdits() ;
+//$demon->fixFauxTemplateReplacements() ;
+
+$demon->set_debug_mode(8);
+
+   // test
+#   $demon->debug_run0();
+   //\\\\\\
+
+$demon->debug("Performing edits...");
+$demon->performEdits() ;
+while ( 1 ) {
+	$demon->debug("Calling run loop...");
+	$demon->run() ;
+	$demon->debug("Sleeping 30...");
+	sleep ( 30 ) ;
+}
+
+?>