Commits

Palmer, 2E0EOL committed 1dc544a Merge

Merge develop (forking support).

  • Participants
  • Parent commits 0b6b027, fb06a3e

Comments (0)

Files changed (2)

 use Getopt::Std;
 use URI::Escape;
 use IO::Interactive qw(is_interactive);
+use POSIX ":sys_wait_h";
 use strict;
 use warnings;
 use diagnostics;
 	return $uri_parts[-1];
 }
 
-sub ReadFeed($$$)
+sub ReadFeed($$)
 {
 	my @entries = ( );
-	my ( $Feeds, $Url, $Name ) = @_;
+	my ( $Feeds, $Feed ) = @_;
+	my ( $Url, $Name ) = ( $Feed->{rss}, $Feed->{name} );
 	my $feed = XML::Feed->parse(URI->new($Url));
 	if ( !$feed ) {
 		printf(STDERR "Stream %s error: %s\n", $Name, $@);
 	return $V;
 }
 
+sub Child($$)
+{
+	my ( $Feeds, $Feed ) = @_;
+	my @entries = ReadFeed($Feeds, $Feed);
+	foreach my $entry ( @entries ) {
+		print STDERR 'Processing entry: ' . Dumper $entry if ( $Debug );
+		DownloadStream($Feeds, $entry, $Feed->{name});
+	}
+}
+
 sub main()
 {
+	my $proctitle = $0; # Save original process title
 	my $conf;
 	my @entries = ( );
 	my @defKeys;
 	my %feeds = ( );
 	my %opts = ( );
+	my %pids;
 	my $confSeen = 0;
 	my @confFiles = (
 		'dlpodget.rc',
 		}
 	}
 
+	$0 = "$proctitle [MASTER]" if ( $feeds{_main}->{MAXCHILDREN} != 0 );
 	foreach my $feedName ( keys(%feeds) ) {
+		my $reaped_pid;
+		my $child_limit_reached = 0;
 		my $feed = $feeds{$feedName};
 		next if ( !$feed->{enable} );
 		next if ( !$feed->{download} );
-		@entries = ReadFeed(\%feeds, $feed->{rss}, $feedName);
-		foreach my $entry ( @entries ) {
-			print STDERR 'Processing entry: ' . Dumper $entry if ( $Debug );
-			DownloadStream(\%feeds, $entry, $feedName);
+		$feed->{name} = $feedName;
+		if ( $feeds{_main}->{MAXCHILDREN} == 0 ) { # Master performs downloads
+			$0 = "$proctitle [$feedName]";
+			Child(\%feeds, $feed);
+			next;
+		} elsif ( $feeds{_main}->{MAXCHILDREN} < 0 || scalar(keys(%pids)) < $feeds{_main}->{MAXCHILDREN} ) {
+			my $pid = fork();
+			die "cannot fork: $!" if ( !defined($pid) );
+			if ( $pid == 0 ) { # Child process
+				$0 = "$proctitle [$feedName]";
+				Child(\%feeds, $feed);
+				exit(0); # Children should not return
+			} else {
+				$pids{$pid} = $feedName;
+			}
+		} else {
+			warn "Not enough children, waiting.";
+			$child_limit_reached = 1;
 		}
+
+		if ( $child_limit_reached ) {
+			$reaped_pid = wait();
+		} else {
+			$reaped_pid = waitpid(-1, WNOHANG); # Any children finished?
+			next if ( $reaped_pid <= 0 ); # No, carry on.
+		}
+
+		# A child returned, remove it from the PID list
+		die 'Internal error' if ( !exists($pids{$reaped_pid}) );
+		delete($pids{$reaped_pid});
+
+		redo if ( $child_limit_reached );
+	}
+	while ( scalar(keys(%pids)) ) {
+		my $pid = wait();
+		last if ( $pid == -1 );
+		delete($pids{$pid});
 	}
 
 	return 0;
 ;
 ; More generally, we have a default section, which contains the following options:
 ;
-;   enable    = <1|0>, default 1, set to 0 for a convenient way to ignore the entire file.
-;   localpfx  = <prefix which will be exported to sections as $LOCALPFX>
-;   noop      = <1|0>, default 0, set to 1 to disable modification of local files.
+;   enable      = <1|0>, default 1, set to 0 for a convenient way to ignore the entire file.
+;   localpfx    = <prefix which will be exported to sections as $LOCALPFX>
+;   noop        = <1|0>, default 0, set to 1 to disable modification of local files.
+;   maxchildren = <0-n>, default 0, max concurrent downloads. nb. setting this to 1
+;                 is less efficient than 0, since a dedicated child is cloned for each feed.
+;                 -1 means no limit (no more than the number of feeds in the configuration).
 
 enable    = 1
 noop      = 0
 debug     = 0
+maxchildren  = 5
 
 [paths]
 root      = $HOME/podcasts