pastebin - collaborative debugging tool
kpaste.net RSS

Difference between
modified post f0b64 by Anonymous on Sat 22nd Sep 2012 13:05
original post 971d62 by Anonymous on Sat 22nd Sep 2012 09:46
Download diff
Show old version | new version | both versions

    
11
#!/usr/bin/perl
22
33
use 5.12.3;
44
use strict;
55
use warnings;
66
use Cwd qw(abs_path cwd);
77
use Digest::MD5 qw(md5_hex);
88
use IO::Handle qw(autoflush);
99
use File::Basename qw(basename dirname);
1010
#use File::Slurp qw(read_file);
1111
use diagnostics;
1212
1313
use threads qw(yield);
1414
use threads::shared;
1515
use Thread::Queue;
1616
use Thread::Semaphore;
1717
#use Fcntl qw(:flock);
1818
use POSIX qw(SIGINT);
1919
use POSIX qw(ceil);
2020
2121
chomp(my $cores = `grep -c ^processor /proc/cpuinfo`);
2222
2323
my (@lib, $mode);
2424
2525
# Path to and name of log file to be used for logging.
2626
my $logf = "$ENV{HOME}/md5db.log";
2727
2828
# Delimiter used for database
2929
my $delim = "\t\*\t";
3030
3131
# Array for storing the actual arguments used by the script internally.
3232
# Might be useful for debugging.
3333
my @cmd = (basename($0));
3434
3535
# Name of database file.
3636
my $db = 'md5.db';
3737
3838
# Clear screen command.
3939
my $clear = `clear && echo`;
4040
4141
# Creating a hash that will store the names of files that are
4242
# too big to fit into RAM. We'll process them last.
43-
my %large;
43+
my %large :shared;
4444
4545
# Creating a few shared variables.
4646
# %err will be used for errors
4747
# $n will be used to count the number of files processed
4848
# %md5h is the database hash
4949
my %err :shared;
5050
my $n :shared = 0;
5151
my %md5h :shared;
5252
my %file_contents :shared;
5353
my $stopping :shared = 0;
5454
my $file_stack :shared = 0;
5555
my $busy :shared = 0;
56-
my @parts :shared = (0) x 2;
5756
58-
my $disk_size = 500000000;
57+
my $disk_size = 1000000000;
5958
6059
# This will be used to control access to the logger subroutine.
6160
my $semaphore = Thread::Semaphore->new();
6261
6362
POSIX::sigaction(SIGINT, POSIX::SigAction->new(\&handler))
6463
|| die "Error setting SIGINT handler: $!\n";
6564
6665
# Creating a custom POSIX signal handler.
6766
# First we create a shared variable that will work as a SIGINT switch.
6867
# Then we define the handler subroutine.
6968
# If ^C is pressed for the first time,
7069
# trip the switch and let the current threads finish.
7170
# If ^C is pressed a second time, quit directly.
7271
# Each subroutine to be used for starting threads will have to
7372
# take notice of the state of the $saw_sigint variable.
7473
my $saw_sigint :shared = 0;
7574
sub handler {
7675
	if ($saw_sigint == 1) {
7776
		logger('int', $n);
7877
		hash2file();
7978
		exit;
8079
	} else { $saw_sigint = 1; }
8180
}
8281
8382
# Open file handle for the log file
8483
open(my $LOG, '>>', $logf) or die "Can't open '$logf': $!";
8584
8685
# Make the $LOG file handle unbuffered for instant logging.
8786
$LOG->autoflush(1);
8887
8988
# Duplicate STDERR as a regular file handle
9089
open(my $SE, ">&STDERR") or die "Can't duplicate STDERR: $!";
9190
9291
### Subroutine for printing usage instructions
9392
sub usage {
9493
9594
	my $s = basename($0);
9695
9796
	say <<"HELP"
9897
Usage: $s [options] [directory 1] .. [directory N]
9998
10099
	-help Print this help message.
101100
101+
	-double Check database for files that have identical
102104
	-import Import MD5 sums to the database from already existing
103105
	\*.MD5 files in each directory.
104106
105107
	-index Index new files in each directory.
106108
107109
	-test Test the MD5 sums of the files in the database to see if
108110
	they've changed.
109111
110112
HELP
111113
}
112114
113115
# This loop goes through the argument list as passed to the script
114116
# by the user when ran.
115117
foreach my $arg (@ARGV) {
116118
117119
	# If argument starts with a dash '-', interprete it as an option
118120
	if ($arg =~ /^-/) {
119121
120122
		given ($arg) {
123+
			# When '-double', set script mode to 'double', and call
121128
122129
			# When '-import', set script mode to 'import', and call
123130
			# the md5import subroutine later.
124131
			when (/^-import$/) {
125132
				if (!$mode) { push(@cmd, $arg); $mode = 'import'; }
126133
			}
127134
128135
			# When '-help', set script mode to 'help', and print
129136
			# usage instructions later.
130137
			when (/^-help$/) {
131138
				if (!$mode) { push(@cmd, $arg); $mode = 'help'; }
132139
			}
133140
134141
			# When '-index', set script mode to 'index', and call
135142
			# the md5index subroutine later.
136143
			when (/^-index$/) {
137144
				if (!$mode) { push(@cmd, $arg); $mode = 'index'; }
138145
			}
139146
140147
			# When '-test', set the script mode to 'test', and call
141148
			# the md5test subroutine later.
142149
			when (/^-test$/) {
143150
				if (!$mode) { push(@cmd, $arg); $mode = 'test'; }
144151
			}
145152
		}
146153
	# If argument is a directory, include it in the @lib array
147154
	} elsif (-d $arg) { push(@lib, $arg); push(@cmd, $arg); }
148155
}
149156
150157
# If no switches were used, print usage instructions
151-
if (!@lib ||
158+
if (!@lib || !$mode || $mode eq 'help')
154159
	{ usage; exit; }
155160
156161
#say "@cmd\n";
157162
158163
# Subroutine for controlling the log file
159164
# Applying a semaphore so multiple threads won't try to
160165
# access it at once, just in case ;-)
161166
sub logger {
162167
163168
	$semaphore->down();
164169
165170
	my($arg, $sw, @fn, $n);
166171
167172
	# Creating a variable to hold the current time.
168173
	my $now = localtime(time);
169174
170175
	# Array of accepted switches to this subroutine
171176
	my @larg = qw{start int gone corr diff end};
172177
173178
	# Loop through all the arguments passed to this subroutine
174179
	# Perform checks that decide which variable the arguments are to
175180
	# be assigned to.
176181
	CHECK: while (@_) {
177182
			$arg = shift(@_);
178183
179184
			# If $arg is a switch, set the $sw variable and start
180185
			# the next iteration of the CHECK loop.
181186
			foreach (@larg) {
182187
				if ($_ eq $arg) { $sw = $arg; next CHECK; }
183188
			}
184189
185190
			# If $arg is a number assign it to $n, if it's a file
186191
			# add it to @fn.
187192
			if ($arg =~ /^[0-9]*$/) { $n = $arg; }
188193
			else { push(@fn, $arg); }
189194
	}
190195
	given ($sw) {
191196
		# Starts writing the log.
192197
		when ('start') {
193198
			say $LOG "\n**** Logging started on $now ****\n";
194199
			say $LOG "Running script in '$mode' mode.\n";
195200
		}
196201
		# When the script is interrupted by user pressing ^C,
197202
		# say so in STDOUT, close the log.
198203
		when ('int') {
199204
			say "\nInterrupted by user!\n";
200205
			say $LOG $n . " file(s) were tested.";
201206
			say $LOG "\n**** Logging ended on $now ****\n";
202207
			close $LOG or die "Can't close '$LOG': $!";
203208
		}
204209
		# Called when file has been deleted or moved.
205210
		when ('gone') {
206211
			say $LOG $fn[0] . "\n\t" . "has been (re)moved.\n";
207212
			$err{$fn[0]} = "has been (re)moved.\n";
208213
		}
209214
		# Called when file has been corrupted.
210215
		when ('corr') {
211216
			say $LOG $fn[0] . "\n\t" .
212217
			"has been corrupted.\n";
213218
			$err{$fn[0]} = "has been corrupted.\n";
214219
		}
215220
		when ('diff') {
216221
			say $LOG $fn[0] . "\n\t" .
217222
				"doesn't match the hash in database.\n";
218223
			$err{$fn[0]} = "doesn't match the hash in database.\n";
219224
		}
220225
		# Called when done, and to close the log.
221226
		# If no errors occurred write "Everything is OK!" to the log.
222227
		# If errors occurred print the %err hash.
223228
		# Either way, print number of files processed.
224229
		when ('end') {
225230
			if (!%err) {
226231
				say $LOG "\nEverything is OK!\n";
227232
			} else {
228233
				say "\n**** Errors Occurred ****\n";
229234
				foreach my $fn (sort keys %err) {
230235
					say $SE $fn . "\n\t" . $err{$fn};
231236
				}
232237
			}
233238
234239
			say $LOG $n . " file(s) were tested.\n" if ($n);
235240
			say $LOG "\n**** Logging ended on $now ****\n";
236241
			close $LOG or die "Can't close '$LOG': $!";
237242
		}
238243
	}
239244
	$semaphore->up();
240245
}
241246
242247
243248
# Subroutine for reading a database file into the database hash.
244249
# This is the first subroutine that will be executed and all others
245250
# depend upon it, cause without it we don't have a
246251
# database hash to work with.
247252
sub file2hash {
248253
249254
	# The format string which is used for parsing the database file
250255
	my $format = qr/^.*\t\*\t[[:alnum:]]{32}$/;
251256
	my (@dbfile, @gone);
252257
253258
	# Open the database file and read it into the @dbfile variable
254259
	open(MD5DB, '<', $db) or die "Can't open '$db': $!";
255260
	chomp (@dbfile = (<MD5DB>));
256261
	close(MD5DB) or die "Can't close '$db': $!";
257262
258263
	# Loop through all the lines in the database file and split
259264
	# them before storing in the database hash.
260265
	# Also, print each line to STDOUT for debug purposes
261266
	foreach my $line (@dbfile) {
262267
263268
		# If current line matches the proper database file format,
264269
		# continue.
265270
		if ($line =~ /$format/) {
266271
267272
			my ($fn, $hash) = (split(/\Q$delim/, $line));
268273
269274
			# If $fn is a real file and not already in the hash,
270275
			# continue.
271276
			if (-f $fn && ! $md5h{$fn}) {
272277
					$md5h{$fn} = $hash;
273278
					say "$fn". "$delim" . "$hash";
274279
275280
			# Saves the names of deleted or moved files in '@gone'
276281
			# for printing at the end of this subroutine.
277282
			} else { push(@gone, $fn); }
278283
		}
279284
	}
280285
281286
	# Clears the screen, thereby scrolling past the database file print
282287
	print $clear;
283288
284289
	# Loops through the @gone array and logs every file name
285290
	# that's been deleted or moved.
286291
	foreach my $fn (@gone) { logger('gone', $fn); }
287292
}
288293
289294
# Subroutine for printing the database hash to the database file
290295
sub hash2file {
291296
292297
	open(MD5DB, '>', "$db") or die "Can't open '$db': $!";
293298
	# Loops through all the keys in the database hash and prints
294299
	# the entries (divided by the $delim variable) to the database file.
295300
	foreach my $k (sort keys %md5h) {
296301
		say MD5DB $k . $delim . $md5h{$k};
297302
	}
298303
	close(MD5DB) or die "Can't close '$db': $!";
299304
}
300305
301306
# Subroutine for finding files
302307
# Finds all the files inside the directory name passed to it,
303308
# and sorts the output before storing it in the @files array.
304309
sub getfiles {
305310
306311
	my $dn = shift;
307312
	my @files;
308313
309314
	open(FIND, '-|', qq(find -L "$dn" -type f -name "*"))
310315
	or die "Can\'t run 'find': $!";
311316
	while (my $fn = (<FIND>)) {
312317
			chomp($fn);
313318
			$fn =~ s($dn/)();
314319
			push(@files, $fn) if (-f $fn && $fn ne basename($db));
315320
	}
316321
	close(FIND) or die "Can't close 'find': $!";
317322
	return @files;
318323
}
319324
320-
# Subroutine for finding and parsing *.MD5 files, adding the hashes
325+
sub md5double {
326326
327-
	my ($fn, $hash, @fields, @lines);
327+
	if (!keys(%md5h)) {
328+
		say "No database file. Run the script in 'index' mode first\n" .
328332
329-
	# The format string which is used for parsing the *.MD5 files.
333+
	# Loop through the %md5h hash and save the checksums as keys in a
330-
	my $format = qr/^[[:alnum:]]{32}\s\*.*/;
334+
	# new hash called %exists. Each of those keys will hold an
331-
335+
	# anonymous array with the matching file names.
332-
	# Loop through the @files array.
336+
	my %exists;
333-
	while (my $md5fn = $q->dequeue_nb()) {
337+
	foreach my $fn (keys(%md5h)) {
338+
		my $hash = $md5h{$fn};
334345
335-
		# If the file extension is *.MD5 in either upper- or
346+
	# Loop through the %exists hash and print files that are identical,
336-
		# lowercase, continue.
347+
	# if any.
337-
		if ($md5fn =~ /.md5$/i) {
348+
	foreach my $hash (keys(%exists)) {
349+
		if (scalar(@{$exists{"$hash"}}) > 1) {
338358
339-
			# Open the *.MD5 file and read its contents to the
359+
# Subroutine for finding and parsing *.MD5 files, adding the hashes
340-
			# @lines array.
360+
# to the database hash and thereby also to the file.
341-
			open(MD5, '<', $md5fn) or die "Can't open '$md5fn': $!";
361+
# It takes 1 argument:
342-
			chomp(@lines = (<MD5>));
362+
# (1) the thread queue
343-
			close(MD5) or die "Can't close '$md5fn': $!";
363+
sub md5import {
344364
345-
			# Loop to check that the format of the *.MD5 file really
365+
	my $md5fn = shift;
348366
349-
				# If format string matches the line(s) in the *.MD5
367+
	my ($fn, $hash, @fields, @lines);
352368
353-
					# Split the line so that the hash and file name go
369+
	# The format string which is used for parsing the *.MD5 files.
354-
					# into @fields array.
370+
	my $format = qr/^[[:alnum:]]{32}\s\*.*/;
363371
364-
					if ($path eq '.') { $fn = basename($fields[1]); }
372+
	# If the file extension is *.MD5 in either upper- or
365-
					else { $fn = dirname($md5fn)
373+
	# lowercase, continue.
366-
								. '/' . basename($fields[1]); }
374+
	if ($md5fn =~ /.md5$/i) {
367375
368-
					# Convert CR+LF newlines to proper LF to avoid
376+
		# Open the *.MD5 file and read its contents to the
369-
					# identical file names from being interpreted as
377+
		# @lines array.
370-
					# different.
378+
		open(MD5, '<', $md5fn) or die "Can't open '$md5fn': $!";
371-
					$fn =~ s/\r//;
379+
		chomp(@lines = (<MD5>));
380+
		close(MD5) or die "Can't close '$md5fn': $!";
372381
373-
					# Unless file name already is in the database hash,
382+
		# Loop to check that the format of the *.MD5 file really
374-
					# print a message, add it to the hash.
383+
		# is correct before proceeding.
375-
					if (! $md5h{$fn} && -f $fn) {
384+
		foreach my $line (@lines) {
376385
377-
						say "$fn" . "\n\t" .
386+
			# If format string matches the line(s) in the *.MD5
378-
						"Imported MD5 sum from '" .
387+
			# file, continue.
379-
						basename($md5fn) .
388+
			if ($line =~ /$format/) {
381389
382-
						$md5h{$fn} = $hash;
390+
				# Split the line so that the hash and file name go
391+
				# into @fields array.
383400
384-
					# If file name is not a real file, write to
401+
				if ($path eq '.') { $fn = basename($fields[1]); }
385-
					# the log.
402+
				else { $fn = dirname($md5fn)
386-
					# If file name is in database hash but the
403+
				. '/' . basename($fields[1]); }
387-
					# MD5 sum from the MD5 file doesn't match,
404+
388-
					# print to the log.
405+
				# Convert CR+LF newlines to proper LF to avoid
389-
					} elsif (! -f $fn) { logger('gone', $fn); }
406+
				# identical file names from being interpreted as
390-
					elsif ($md5h{$fn} ne $hash)
407+
				# different.
391-
					{ logger('diff', $md5fn); }
408+
				$fn =~ s/\r//;
397409
398-
sub md5sum {
410+
				# Unless file name already is in the database hash,
399-
	my $fn = shift;
411+
				# print a message, add it to the hash.
400-
	my $last = $parts[1];
412+
				if (! $md5h{$fn} && -f $fn) {
401413
402-
	# If $last is set it means we're dealing with a split file, so
414+
					say "$fn" . "\n\t" .
403-
	# set $busy to 1 for the other threads to fall asleep. We need to
415+
					"Imported MD5 sum from '" .
404-
	# stay in this thread when we're splitting files. So we create an
416+
					basename($md5fn) .
405-
	# 'until' loop which will process all the parts until the last one.
417+
					"'.\n";
406-
	if ($last) {
418+
407-
		$busy = 1;
419+
					$md5h{$fn} = $hash;
408-
		my $md5 = Digest::MD5->new;
420+
409-
		my $done = 0;
421+
					# If file name is not a real file, write to
410-
		until ($done == $last) {
422+
					# the log.
411-
			if ($file_contents{$fn}) {
423+
					# If file name is in database hash but the
412-
				$md5->add($file_contents{$fn});
424+
					# MD5 sum from the MD5 file doesn't match,
413-
				lock($file_stack);
425+
					# print to the log.
414-
				$file_stack -= length($file_contents{$fn});
426+
				} elsif (! -f $fn) { logger('gone', $fn); }
415-
				lock(%file_contents);
427+
				elsif ($md5h{$fn} ne $hash)
416-
				delete($file_contents{$fn});
428+
				{ logger('diff', $md5fn); }
418429
			}
420430
		}
421-
		my $hash = $md5->hexdigest;
431+
	}
431432
	}
433+

Submit a correction or amendment below (click here to make a fresh posting)
After submitting an amendment, you'll be able to view the differences between the old and new posts easily.

Syntax highlighting:

To highlight particular lines, prefix each line with {%HIGHLIGHT}




All content is user-submitted.
The administrators of this site (kpaste.net) are not responsible for their content.
Abuse reports should be emailed to us at