1. Michael Granger
  2. newznabr

Commits

Michael Granger  committed 16d79a3

Checkpoint commit

  • Participants
  • Parent commits ce6385a
  • Branches default

Comments (0)

Files changed (14)

File data/newznabr/apps/newznabr-rest-service

View file
 class NewzNabr::RESTService < Strelka::App
 	extend Configurability
 
+	# Strelka App ID
+	ID = 'newznabr-rest-service'
+
+	# Valid search result limits
+	VALID_LIMITS = [ 10, 25, 50, 100 ]
+
+
 	# Configurability API -- configure with the 'services' section
 	config_key :services
 
+	# Loggability API -- log to the :newznabr logger
+	log_to :newznabr
 
-	# Strelka App ID
-	ID = 'newznabr-rest-service'
 
 	#
 	# Parameter validation
 	#
 
 	param :last_updated, lambda {|string| Time.parse(string) }
-	param :keywords, /^(?<keywords>\w+(?:[ +]+\w+)*)$/, "Search keywords"
+	param :keywords, /^(?<keywords>\w+(?:[ +]+\w+)*|\*)$/, "Search keywords"
 
 
 	#
 
 	# api_description "A REST service for a NewzNabr index"
 
-	FT_QUERY = (<<-END_QUERY).strip.gsub( /\n\s*/, ' ' )
-		to_tsvector('english', searchname)
-		@@
-		to_tsquery('english', ?)
-	END_QUERY
+	get '/releases/search' do |req|
+		req.params.add :limit, :integer
+		req.params.add :offset, :integer
+		req.params.add :order, /^(?:name|fromname|postdate|adddate|tvairdate)$/, :multiple
 
-	get '/releases/search' do |req|
-		keywords = req.params[:keywords].split( /[ +]+/ )
-		self.log.debug "Searching for releases with keywords: %p" % [ keywords ]
-		dataset = NewzNabr::Release.dataset.filter( FT_QUERY, keywords.compact.join(' & ') )
+		finish_with HTTP::BAD_REQUEST, req.params.error_messages.join(', ') unless req.params.okay?
+		self.log.debug "Params were: %s" % [ req.params ]
 
-		if limit = req.params[:limit]
-			offset = req.params[:offset]
-			dataset = dataset.limit( limit, offset || 0 )
-		end
-
+		dataset = self.make_search_dataset( req.params )
 		self.log.debug "Query is: %s" % [ dataset.sql ]
 
 		res = req.response
 		return res
 	end
 
+
+	### Return an appropriate Release dataset for the given +params+.
+	def make_search_dataset( params )
+		keywords, limit, offset, order = params.values_at( :keywords, :limit, :offset, :order )
+
+		limit = VALID_LIMITS.first unless VALID_LIMITS.include?( limit )
+		offset = 0 unless offset && (offset % limit).zero?
+
+		dataset = NewzNabr::Release.dataset
+
+		if keywords != '*'
+			keywords = keywords.split( /[ +]+/ )
+			self.log.debug "Searching for releases with keywords: %p" % [ keywords ]
+			query = keywords.compact.join(' & ')
+			dataset = dataset.full_text_search( :searchname, query, language: 'english' )
+		end
+
+		if limit
+			self.log.debug "Applying limit %d" % [ limit ]
+			dataset = dataset.limit( limit, offset )
+		end
+
+		if order
+			order = Array( order ).map( &:to_sym )
+			self.log.debug "Ordering result set by %p" % [ order ]
+			dataset = dataset.order( *order )
+		end
+	end
+
 end # class NewzNabr::RESTService
 
 Encoding.default_internal = Encoding::UTF_8

File data/newznabr/defaultregexes.yml

View file
     :active: true
     :description: A better matcher with part numbers
     :category_id: 
+- !ruby/object:NewzNabr::ReleaseRegex
+  values:
+    :id: 12
+    :groupname: alt.binaries.*
+    :regex: ^\[\d+\]-\[\w+\]-\[#.*?\]-\[\s*(?<name>.*?)-(?<fromname>\S+)\s*\]-\[(?<parts>\d+/\d+)\] "+(?<filename>.*?)"+\s+yEnc$
+    :ordinal: 52
+    :active: true
+    :description: Bracket group "teevee" style.
+    :category_id: 

File data/newznabr/static/css/app.css

View file
   overflow: scroll;
 }
 
+table#release-results {
+	width: 100%;
+}
 
 #dashboard-searchbox {
 	padding-top: 4em;

File data/newznabr/static/js/newznabr.js

View file
 /**
- * Curator Client-Side AngularJS Application
+ * Newznabr Client-Side AngularJS Application
  * $Id$
  */
 'use strict';
 
+/**
+ * Tuning variables
+ */
+
+// Max items to fetch (at a time) for a search.
+var SearchLimit = 100;
 
 /*
     JavaScript Relative Time Helpers
 angular.module('navmenu.components', [])
 
 	/**
-	 * <navmenu /> Directive
-	 * Add this element to a view to build the navigation menu automatically
-	 * out of menuitem members in the $routeProvider.
-	 */
+	* <navmenu /> Directive
+	* Add this element to a view to build the navigation menu automatically
+	* out of menuitem members in the $routeProvider.
+	*/
 	.directive( 'navmenu', function($route) {
 		return {
 			restrict: 'E', // element name
 			templateUrl: '/views/navmenu.html',
 			replace: true,
 
+			// Build the navmenu at link time
 			link: function( $scope, $element, $attrs, $controller ) {
-				console.debug( "Linking the navmenu up. $route is: %o", $route );
-
 				var menuRoutes = [];
 				for ( var o in $route.routes ) {
 					var route = $route.routes[ o ];
 			},
 
 			controller: function( $scope, $element, $location ) {
-				console.debug( "Controller for %o hooked up with scope %o.", $element, $scope );
+				// When the route changes, update the 'active' class on the nav items
 				$scope.$on( '$routeChangeSuccess', function(e, current, prev) {
-					console.debug( "Route changed from %o to %o", prev, current );
 					$scope.menuitems.forEach( function(item) {
-						console.debug( "checking to see if %o is the item for the current route (%o)",
-							item.route, current.$route );
+						if ( typeof item.styleclass != 'Array' ) {
+							item.styleclass = [];
+						}
 
-						if ( current.$route && (item.route.controller == current.$route.controller) ) {
-							console.debug( "  yep." );
-							item.styleclass += ['current'];
+						if ( current.$$route && (item.route.controller == current.$$route.controller) ) {
+							item.styleclass += ['active'];
 						} else {
-							console.debug( "  nope." );
+							item.styleclass -= ['active'];
 						}
 					});
 				});
 		};
 	});
 
+
 angular.module('navmenu', ['navmenu.components'] );
 
 
 
 /*
- * Curator application
+ * Newznabr application
  */
 
 // Controllers
-angular.module('newznabr.controllers', ['newznabr.services']).
+angular.module('newznabr.controllers', ['infinite-scroll', 'newznabr.services']).
 	controller('DashboardController', 
 		function dashboardController($scope, $log, Release, Binary) {
 			$log.debug( "Controller for Dashboard running." );
-			$scope.viewname = 'Dashboard';
-			$scope.releases = Release.query({ limit: 15 });
+
+			$scope.viewname    = 'Dashboard';
+			$scope.releases    = [];
 			$scope.searchterms = '';
-			$scope.search = function search(searchterms) {
-				$log.log( "Search for %o", searchterms );
-				$scope.releases = Release.query({ keywords: searchterms, limit: 15 });
+			$scope.offset      = 0;
+			$scope.order       = 'postdate';
+			$scope.busy        = false;
+			$scope.done        = false;
+
+			$scope.search = function search() {
+				if ( $scope.searchterms === '' ) $scope.searchterms = '*';
+				$log.log( "Search for %o", $scope.searchterms );
+				$scope.busy = true;
+				$scope.done = false;
+				$scope.offset = 0;
+
+				$scope.releases =
+					Release.search({
+						keywords: $scope.searchterms,
+						limit: SearchLimit,
+						order: $scope.order },
+						function() { $scope.busy = false });
 			};
+
+			$scope.loadMoreResults = function loadMoreResults() {
+				if ( $scope.searchterms === '' || $scope.busy || $scope.done ) return;
+				$scope.busy = true;
+
+				console.debug( "Loading more results..." );
+
+				// If we didn't load SearchLimit results last time, we're done
+				if ( (new_offset % SearchLimit) !== 0 ) {
+					$scope.done = true;
+					$scope.busy = false;
+					console.debug( "Done: loaded %d results.", $scope.releases.length );
+					return;
+				}
+				var new_offset = $scope.releases.length;
+				$scope.offset = new_offset;
+
+				Release.search({
+					keywords: $scope.searchterms,
+					limit:    SearchLimit,
+					offset:   $scope.offset,
+					order:    $scope.order },
+					function(results) {
+						if ( results.length === 0 ) {
+							$scope.done = true;
+						} else {
+							results.forEach( function(result) {
+								$scope.releases.push(result);
+							});
+						}
+
+						$scope.busy = false;
+					});
+			};
+
+			$scope.orderBy = function orderBy( field ) {
+				console.debug( "Ordering by %s", field );
+				$scope.order = field;
+				$scope.search();
+			}
 		}
 	).
 	controller('SettingsController',
 			$log.log( "Controller for Settings running." );
 			$scope.viewname = 'Settings';
 
-			Group.query(function(groups) {
-				// Split the fetched groups up between the active and inactive
-				// resources.
-				$scope.activegroups = groups.filter(function(g) { return g.active; } );
-				$scope.inactivegroups = groups.filter(function(g) { return !g.active; } );
-			});
-
+			$scope.groups = Group.query();
 			$scope.regexes = ReleaseRegexes.query();
 
-			$scope.activate = function activate( group ) {
-				group.active = true;
-				group.$update( function() {
-					var i = $scope.inactivegroups.indexOf( group );
-					$scope.inactivegroups.splice( i, 1 );
-					$scope.activegroups.push( group );
-				});
-			};
-			$scope.deactivate = function deactivate( group ) {
-				group.active = false;
-				group.$update( function() {
-					var i = $scope.activegroups.indexOf( group );
-					$scope.activegroups.splice( i, 1 );
-					$scope.inactivegroups.push( group );
-				});
+			$scope.toggleGroup = function toggleGroup( group ) {
+				group.$save();
 			};
 		}
 	);
 				rel: ''
 			},
 			{
-				update: {method: 'PUT'},
+				save: {method: 'PUT'},
 			}
 		);
 	}).
 			{
 				releaseId: '@id',
 				rel: ''
+			},
+			{
+				search: {
+					method: 'GET',
+					url: '/services/newznabr/v1/releases/search',
+					isArray: true
+				}
 			}
 		);
 	}).

File data/newznabr/static/js/vendor/ng-infinite-scroll.js

View file
+/* ng-infinite-scroll - v1.0.0 - 2013-02-23 */
+var mod;
+
+mod = angular.module('infinite-scroll', []);
+
+mod.directive('infiniteScroll', [
+  '$rootScope', '$window', '$timeout', function($rootScope, $window, $timeout) {
+    return {
+      link: function(scope, elem, attrs) {
+        var checkWhenEnabled, handler, scrollDistance, scrollEnabled;
+        $window = angular.element($window);
+        scrollDistance = 0;
+        if (attrs.infiniteScrollDistance != null) {
+          scope.$watch(attrs.infiniteScrollDistance, function(value) {
+            return scrollDistance = parseInt(value, 10);
+          });
+        }
+        scrollEnabled = true;
+        checkWhenEnabled = false;
+        if (attrs.infiniteScrollDisabled != null) {
+          scope.$watch(attrs.infiniteScrollDisabled, function(value) {
+            scrollEnabled = !value;
+            if (scrollEnabled && checkWhenEnabled) {
+              checkWhenEnabled = false;
+              return handler();
+            }
+          });
+        }
+        handler = function() {
+          var elementBottom, remaining, shouldScroll, windowBottom;
+          windowBottom = $window.height() + $window.scrollTop();
+          elementBottom = elem.offset().top + elem.height();
+          remaining = elementBottom - windowBottom;
+          shouldScroll = remaining <= $window.height() * scrollDistance;
+          if (shouldScroll && scrollEnabled) {
+            if ($rootScope.$$phase) {
+              return scope.$eval(attrs.infiniteScroll);
+            } else {
+              return scope.$apply(attrs.infiniteScroll);
+            }
+          } else if (shouldScroll) {
+            return checkWhenEnabled = true;
+          }
+        };
+        $window.on('scroll', handler);
+        scope.$on('$destroy', function() {
+          return $window.off('scroll', handler);
+        });
+        return $timeout((function() {
+          if (attrs.infiniteScrollImmediateCheck) {
+            if (scope.$eval(attrs.infiniteScrollImmediateCheck)) {
+              return handler();
+            }
+          } else {
+            return handler();
+          }
+        }), 0);
+      }
+    };
+  }
+]);

File data/newznabr/static/js/vendor/ng-infinite-scroll.min.js

View file
+/* ng-infinite-scroll - v1.0.0 - 2013-02-23 */
+var mod;mod=angular.module("infinite-scroll",[]),mod.directive("infiniteScroll",["$rootScope","$window","$timeout",function(i,n,e){return{link:function(t,l,o){var r,c,f,a;return n=angular.element(n),f=0,null!=o.infiniteScrollDistance&&t.$watch(o.infiniteScrollDistance,function(i){return f=parseInt(i,10)}),a=!0,r=!1,null!=o.infiniteScrollDisabled&&t.$watch(o.infiniteScrollDisabled,function(i){return a=!i,a&&r?(r=!1,c()):void 0}),c=function(){var e,c,u,d;return d=n.height()+n.scrollTop(),e=l.offset().top+l.height(),c=e-d,u=n.height()*f>=c,u&&a?i.$$phase?t.$eval(o.infiniteScroll):t.$apply(o.infiniteScroll):u?r=!0:void 0},n.on("scroll",c),t.$on("$destroy",function(){return n.off("scroll",c)}),e(function(){return o.infiniteScrollImmediateCheck?t.$eval(o.infiniteScrollImmediateCheck)?c():void 0:c()},0)}}}]);

File data/newznabr/static/views/dashboard.html

View file
-<!-- Initial searchbox (no results) -->
-<div class="row" id="dashboard-searchbox" ng-hide="releases">
-	<div class="large-12 columns">
-		<form accept-charset="utf-8">
-			<fieldset class="row collapse">
-			<div class="large-10 columns">
-				<input type="text" placeholder="keywords, titles, etc." ng-model="searchterms">
-			</div>
-			<div class="large-2 columns">
-				<button ng-click="search(searchterms)" class="button prefix">Search</button>
-			</div>
-			</fieldset>
-		</form>
-	</div>
-</div>
-  
 <!-- Searchbox+Results -->
-<div class="row" ng-show="releases">
+<div class="row">
 	<div class="large-12 columns">
 
 		<form accept-charset="utf-8">
 			<div class="large-10 columns">
 				<input type="text" placeholder="keywords, titles, etc." ng-model="searchterms">
 			</div>
+
 			<div class="large-2 columns">
 				<button ng-click="search(searchterms)" class="button prefix">Search</button>
 			</div>
+			
 			</fieldset>
 		</form>
 
-		<table id="release-results">
+		<table id="release-results" ng-hide="releases.length === 0">
 			<thead>
 				<tr>
-				  <th>Name</th>
-				  <th>From</th>
-				  <th>Date</th>
+				  <th><a ng-click="orderBy('name')">Name</a></th>
+				  <th><a ng-click="orderBy('fromname')">From</a></th>
+				  <th><a ng-click="orderBy('postdate')">Date</a></th>
 				  <th>Parts</th>
 				</tr>
 			</thead>
-			<tbody>
+			<tbody infinite-scroll="loadMoreResults()">
 				<tr ng-repeat="release in releases">
-				  <td>{{release.name}}</td>
-				  <td>{{release.fromname}}</td>
-				  <td>{{release.date | reldate}}</td>
-				  <td>{{release.total_parts}}</td>
+				  <td>{{release.searchname}}</td>
+				  <td>{{release.fromname.substr(0,20)}}</td>
+				  <td>{{release.postdate | reldate}}</td>
+				  <td>{{release.totalpart}}</td>
 				</tr>
 			</tbody>
 		</table>
 	
+		
+	
 	</div>
 </div>
     

File data/newznabr/static/views/settings.html

View file
 			</header>
 
 			<form>
-				<fieldset class="activated">
-					<legend>Active</legend>
-					<table class="activated-groups" ng-show="activegroups">
-						<thead><tr><th>Name</th><th>Description</th></tr></thead>
+				<fieldset>
+					<legend>Activate groups by checking the checkbox.</legend>
+					<table class="groups" ng-show="groups.$resolved">
+						<thead><tr><th>Active?</th><th>Name</th><th>Description</th></tr></thead>
 						<tbody>
-						<tr ng-repeat="group in activegroups | orderBy:'name'"
-							ng-click="deactivate(group)">
-							<td>{{group.name}}</td>
-							<td>{{group.description}}</td>
-						</tr>
-						</tbody>
-					</table>
-				</fieldset>
-
-				<fieldset class="deactivated">
-					<legend>Deactivated</legend>
-					<table class="deactivated-groups" ng-show="inactivegroups">
-						<thead><tr><th>Name</th><th>Description</th></tr></thead>
-						<tbody>
-						<tr ng-repeat="group in inactivegroups | orderBy:'name'"
-							ng-click="activate(group)">
+						<tr ng-repeat="group in groups | orderBy:'name'">
+							<td><input type="checkbox" name="active" ng-model="group.active" 
+								 ng-change="toggleGroup(group)"/></td>
 							<td>{{group.name}}</td>
 							<td>{{group.description}}</td>
 						</tr>

File data/newznabr/templates/layout.tmpl

View file
 	<script src="/js/vendor/angular.min.js" defer="defer"></script>
 	<script src="/js/vendor/angular-resource.min.js" defer="defer"></script>
 	<script src="/js/vendor/angular-sanitize.min.js" defer="defer"></script>
+	<script src="/js/vendor/ng-infinite-scroll.js" defer="defer"></script>
 
     <?subscribe jslibs ?>
   

File etc/mongrel.rb

View file
 	bind_addr '0.0.0.0'
 	port 8223
 
-	default_host 'pendor.local'
+	default_host 'yevaud.local'
 
-	host 'pendor.local' do
+	host 'yevaud.local' do
 		Pathname.glob( (datadir + 'static/*').to_s ).each do |subdir|
 			next unless subdir.directory?
 			puts "  adding static content route for: #{subdir}"
 
 		route '/favicon.ico($)', directory( datadir + 'static/img/', 'favicon.ico' )
 
-		route '/services/newznabr/v1', handler( 'tcp://127.0.0.1:29001', 'newznabr-rest-service' )
-		route '/api', handler( 'tcp://127.0.0.1:29003', 'newznab-service' )
-		route '/', handler( 'tcp://127.0.0.1:29005', 'newznabr' )
+		route '/services/newznabr/v1', handler( 'tcp://127.0.0.1:39001', 'newznabr-rest-service' )
+		route '/api', handler( 'tcp://127.0.0.1:39003', 'newznab-service' )
+		route '/', handler( 'tcp://127.0.0.1:39005', 'newznabr' )
 	end
 
 end

File lib/newznabr/binary.rb

View file
 
 	##
 	# The Release this binary is a part of
-	many_to_one :releases, class_name: 'NewzNabr::Release'
+	many_to_one :release, class_name: 'NewzNabr::Release'
 
 
 	### Create or fetch a Binary for the given +group+ from the data in the given

File lib/newznabr/category.rb

View file
 	subset :toplevel, :parent_id => nil
 
 
+	### Return the most-applicable Category for a release associated with the specified
+	### +binary+ matched by +release_regex+. 
+	def self::detect( binary, release_regex )
+		# Hard-coded for now
+		return self.
+			filter( parent_id: nil ).
+			filter( name: 'Other' ).first.
+			subcategories_dataset.filter( name: 'Misc' ).first
+	end
+
+
 	### Return a string describing the category in relation to its
 	### parents (if it has any); the +separator+ will be used between
 	### the levels of the hierarchy.

File lib/newznabr/command.rb

View file
 			$stderr.puts "Looking at RRE: %s" % [ rre.regex ]
 			rre.matching_binaries.each do |binary, match|
 				self.log.debug "  match: %p" % [ match ]
-				named_captures = Hash[ [match.names, match.captures].transpose ]
-				rel = NewzNabr::Release.from_regexp_match( rre, named_captures )
+				named_captures = Hash[ [match.names.map(&:to_sym), match.captures].transpose ]
+				rel = NewzNabr::Release.from_regexp_match( rre, binary, named_captures )
 				self.prompt.say "  %s" % [ rel ]
 			end
 		end

File lib/newznabr/release.rb

View file
 # -*- ruby -*-
 #encoding: utf-8
 
+require 'securerandom'
+
 require 'newznabr' unless defined?( NewzNabr )
 require 'newznabr/model' unless defined?( NewzNabr::Model )
 
 		int       :passwordstatus, null: false, default: 0
 
 		timestamp :postdate, default: nil
-		timestamp :adddate, default: nil
+		timestamp :adddate, default: Sequel.function(:now)
 		timestamp :tvairdate
 
 		int       :imdb_id
 			on_update: :cascade,
 			on_delete: :set_default,
 			default: 0
-		foreign_key :group_id, :newznabr__groups,
-			on_delete: :cascade,
-			on_update: :cascade
 	end
 
 
 	#
 
 	##
-	# The Group this release is in
-	many_to_one :groups, class_name: 'NewzNabr::Group'
+	# The Binaries this release is composed of
+	one_to_many :binaries, class_name: 'NewzNabr::Binary'
 
 	##
 	# The ReleaseRegex which matched this release
 
 	### Find an existing release matched by the specified +release_regex+ with the
 	### given +captures+, or create a new one.
-	def self::from_regexp_match( release_regex, captures )
-		self.find_or_create( name: captures[:name], fromname: captures[:fromname] ) do |release|
-			release.name     = captures[:name]
-			release.fromname = captures[:fromname]
-			release.regex    = release_regex
-			require 'pry'; binding.pry
+	def self::from_regexp_match( release_regex, binary, captures )
+		release = self.find_or_create( name: captures[:name], fromname: captures[:fromname] ) do |release|
+			self.log.debug "Creating a new release for captures: %p" % [ captures ]
+			part, totalpart = captures[:parts].split( %r|/|, 2 ).map( &:to_i )
+			searchname = captures[:name].gsub( /\P{^Punct}/, ' ' ).squeeze.strip
+
+			release.name       = captures[:name]
+			release.fromname   = captures[:fromname]
+			release.regex      = release_regex
+			release.totalpart  = totalpart
+			release.searchname = searchname
+			release.guid       = SecureRandom.uuid
+			release.postdate   = binary.date
+
+			release.category   = NewzNabr::Category.detect( binary, release_regex )
+			release.regex      = release_regex
+
+			self.log.debug "  new release: %p" % [ release ]
 		end
+
+		release.add_binary( binary )
 	end