Commits

Jason Moiron  committed b398bf9

initial wip commit of ukiyo

  • Participants

Comments (0)

Files changed (6)

File autobuild.sh

+#!/bin/bash
+
+cur=`pwd`
+
+inotifywait -mqr --timefmt '%d/%m/%y %H:%M' --format '%T %w %f' \
+   -e modify ./ | while read date time dir file; do
+    ext="${file##*.}"
+    if [[ "$ext" = "go" ]]; then
+        echo "$file changed @ $time $date, rebuilding..."
+        go build
+    fi
+done
+
+package main
+
+import (
+	"database/sql"
+	"fmt"
+	_ "github.com/mattn/go-sqlite3"
+	"os"
+	"os/user"
+	"path/filepath"
+	"runtime"
+	"strings"
+)
+
+type Xdg struct {
+	DATA_HOME   string
+	CONFIG_HOME string
+	Initialized bool
+}
+
+type Site struct {
+	Name     string
+	Url      string
+	Priority int
+	Updated  int64
+}
+
+type Config struct {
+	DownloadPath string
+	SiteOrder    []string
+	path         string
+}
+
+type Series struct {
+	Name    string
+	Url     string
+	Site    string
+	Key     string
+	Updated int64
+}
+
+var xdg Xdg
+var config Config
+
+var DEFAULT_DOWNLOAD_PATH = expandpath("~/Downloads")
+
+// return whether a path exists
+func exists(path string) bool {
+	_, err := os.Stat(path)
+	if err == nil {
+		return true
+	}
+	if os.IsNotExist(err) {
+		return false
+	}
+	fmt.Printf("%s\n", err)
+	return false
+}
+
+// return a path with posix tilde and ENV expansion
+func expandpath(path string) string {
+	if path[0] == '~' {
+		sep := strings.Index(path, string(os.PathSeparator))
+		if sep < 0 {
+			sep = len(path)
+		}
+		var err error
+		var u *user.User
+		username := path[1:sep]
+		if len(username) == 0 {
+			u, err = user.Current()
+		} else {
+			u, err = user.Lookup(username)
+		}
+		if err == nil {
+			path = filepath.Join(u.HomeDir, path[sep:])
+		}
+	}
+	path = os.ExpandEnv(path)
+	abs, err := filepath.Abs(path)
+	if err != nil {
+		return path
+	}
+	return abs
+}
+
+func getenv(key, default_ string) string {
+	value := os.Getenv(key)
+	if len(value) == 0 {
+		value = default_
+	}
+	return value
+}
+
+// initialize platform dependent data/config paths
+func (x *Xdg) init() {
+	if x.Initialized {
+		return
+	}
+	switch runtime.GOOS {
+	case "linux":
+		x.DATA_HOME = getenv("XDG_DATA_HOME", expandpath("~/.local/share"))
+		x.CONFIG_HOME = getenv("XDG_CONFIG_HOME", expandpath("~/.config"))
+	case "osx":
+		x.DATA_HOME = expandpath("~/Library/Application Support/")
+		x.CONFIG_HOME = expandpath("~/Library/Preferences/")
+	}
+
+	x.Initialized = true
+}
+
+func (c *Config) Open() *sql.DB {
+	db, err := sql.Open("sqlite3", c.path)
+	if err != nil {
+		panic(fmt.Sprintf("%q", err))
+	}
+	return db
+}
+
+func (c *Config) getval(key string) (string, error) {
+	var value string
+	db := c.Open()
+	defer db.Close()
+	row := db.QueryRow("select value from config where key = ?", key)
+	err := row.Scan(&value)
+	return value, err
+}
+
+func (c *Config) init() {
+	var err error
+	if !xdg.Initialized {
+		xdg.init()
+	}
+	configPath := filepath.Join(xdg.CONFIG_HOME, "ukiyo/")
+	os.MkdirAll(configPath, 0755)
+
+	c.path = filepath.Join(configPath, "config.db")
+
+	if !exists(c.path) {
+		c.initDb()
+	}
+
+	c.DownloadPath, err = c.getval("DownloadPath")
+	if err != nil {
+		fmt.Errorf("Could not read key 'DownloadPath' from config: %q\n", err)
+	}
+}
+
+func (c *Config) initDb() {
+	tables := []string{
+		"create table config (key text primary key, value text)",
+		"create table watchlist (key text primary key, name text, chapter text)",
+		"create table sites (name text primary key, url text, priority integer, updated integer default 0)",
+		"create table series (name text, key text, url text primary key, site text, updated integer default 0)",
+		"create table chapters (series text, site text, url text primary key, number text)",
+	}
+	db := c.Open()
+	defer db.Close()
+	// start a transaction;  sqlite is slow as hell without them
+	tx, err := db.Begin()
+	if err != nil {
+		fmt.Printf("Unable to open transaction on config db: %s\n", err)
+		return
+	}
+
+	// create tables
+	for _, t := range tables {
+		_, err := tx.Exec(t)
+		if err != nil {
+			panic(fmt.Sprintf("table panic: %q: %s\n", err, t))
+		}
+	}
+
+	_, err = tx.Exec("insert into config (key, value) values (?, ?)",
+		"DownloadPath", DEFAULT_DOWNLOAD_PATH)
+	if err != nil {
+		panic(fmt.Sprintf("panic: %q\n", err))
+	}
+
+	addSite := "insert into sites (name, url, priority) values (?, ?, ?)"
+	tx.Exec(addSite, "manga-access", "http://www.manga-access.com", 1)
+	tx.Exec(addSite, "mangahere", "http://www.mangahere.com", 2)
+	tx.Exec(addSite, "mangareader", "http://www.mangareader.net", 3)
+	tx.Exec(addSite, "mangafox", "http://www.mangafox.me", 4)
+
+	tx.Commit()
+}
+
+func (c *Config) SetDownloadPath(path string) error {
+	db := c.Open()
+	defer db.Close()
+
+	_, err := db.Exec("update config set value=? where key=?", path, "DownloadPath")
+	return err
+}
+
+func (c *Config) AddSite(name, url string, priority int) error {
+	db := c.Open()
+	defer db.Close()
+
+	_, err := db.Exec("insert into sites (name, url, priority) values (?, ?, ?)",
+		name, url, priority)
+	return err
+}
+
+func (c *Config) RemoveSite(name string) error {
+	db := c.Open()
+	defer db.Close()
+	if len(name) == 0 {
+		return fmt.Errorf("Error: name of site to delete must be provided.")
+	}
+	_, err := db.Exec("DELETE FROM sites WHERE name=?", name)
+	return err
+}
+
+func (c *Config) SetSitePriority(name string, priority int) {
+
+}
+
+func init() {
+	config.init()
+}

File config_test.go

+package main
+
+import (
+	"github.com/kless/sysuser"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func isPosix() bool {
+	username := sysuser.Getuser()
+	_, err := sysuser.LookupUser(username)
+	if err != nil {
+		return false
+	}
+	return true
+}
+
+func TestExpandPath(t *testing.T) {
+	if !isPosix() {
+		return
+	}
+	user, _ := sysuser.LookupUser(sysuser.Getuser())
+	cwd := os.Getenv("PWD")
+	home := user.Dir
+	paths := []string{
+		"foo",
+		"~",
+		"~foo",
+		"~/foo",
+		"foo/~/",
+		"$HOME/foo",
+		"~" + user.Name + "/foo",
+	}
+
+	expanded := []string{
+		filepath.Join(cwd, "foo"), // ./foo
+		home,                      // /home/current/
+		filepath.Join(cwd, "~foo"),   // ./~foo
+		filepath.Join(home, "foo"),   // /home/current/foo
+		filepath.Join(cwd, "foo/~/"), // ./foo/~/
+		filepath.Join(home, "foo"),   // /home/current/foo
+		filepath.Join(home, "foo"),   // /home/current/foo
+	}
+
+	for i, path := range paths {
+		res := expandpath(path)
+		if res != expanded[i] {
+			t.Errorf("%d. Expected '%s' => '%s', got '%s'", i, paths[i], expanded[i], res)
+		}
+	}
+}
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/moovweb/gokogiri"
+	"github.com/moovweb/gokogiri/css"
+	"github.com/moovweb/gokogiri/html"
+	"github.com/moovweb/gokogiri/xml"
+	"github.com/moovweb/gokogiri/xpath"
+	"time"
+	"unsafe"
+)
+
+func tick() { fmt.Printf("%s\n", time.Now().String()) }
+
+// Selectable implements a simple interface which allows to get the inner text
+// of some element as well as run a CSS select on it and get a list of nodes
+type Selectable interface {
+	CssSelect(selector string) []Node
+	Text() string
+}
+
+// A node wrapper, in order to provide a similar interface in the future
+// possibly without gokogiri
+type Node struct {
+	Selectable
+	doc  *html.HtmlDocument
+	ptr  unsafe.Pointer
+	node xml.Node
+}
+
+// A Document wrapper, which can be Freed and Selected, and exposes
+// the root as a Node object with the Root field
+type Document struct {
+	Selectable
+	doc    *html.HtmlDocument
+	docptr unsafe.Pointer
+	Root   Node
+}
+
+// Fill a Node element from a ptr
+func (n *Node) fromPtr(ptr unsafe.Pointer, doc *html.HtmlDocument) {
+	n.ptr = ptr
+	n.doc = doc
+	n.node = xml.NewNode(ptr, doc)
+}
+
+// Fill a Node element from an xml.Node
+func (n *Node) fromNode(node xml.Node, doc *html.HtmlDocument) {
+	n.ptr = node.NodePtr()
+	n.node = node
+	n.doc = doc
+}
+
+func (n *Node) CssSelect(selector string) []Node {
+	xpathexpr := css.Convert(selector, 0)
+	expr := xpath.Compile(xpathexpr)
+	nxp := xpath.NewXPath(n.ptr)
+	defer nxp.Free()
+	nodes := nxp.Evaluate(n.ptr, expr)
+	ret := make([]Node, len(nodes))
+	for i, ptr := range nodes {
+		ret[i].fromPtr(ptr, n.doc)
+	}
+	return ret
+}
+
+func (n *Node) Text() string {
+	return n.node.Content()
+}
+
+func (n *Node) Attr(attr string) string {
+	return n.node.Attr(attr)
+}
+
+func (d *Document) CssSelect(selector string) []Node {
+	return d.Root.CssSelect(selector)
+}
+
+func (d *Document) Text() string {
+	return ""
+}
+
+func (d *Document) FromString(str string) error {
+	buff := bytes.NewBufferString(str)
+	bites := buff.Bytes()
+	return d.FromBytes(bites)
+}
+
+func (d *Document) FromBytes(str []byte) error {
+	doc, err := gokogiri.ParseHtml(str)
+	if err != nil {
+		return err
+	}
+	d.doc = doc
+	d.Root.fromNode(doc.Root(), doc)
+	d.docptr = doc.DocPtr()
+	return nil
+}
+
+func (d *Document) Free() {
+	d.doc.Free()
+}
+package main
+
+import (
+	"fmt"
+	"github.com/jmoiron/go-pkg-optarg"
+	"strconv"
+	"strings"
+)
+
+const VERSION = "0.1a"
+
+type Options struct {
+	Update          bool
+	Help            bool
+	Version         bool
+	Verbose         bool
+	List            bool
+	SetDownloadPath bool
+	Sync            bool
+	// sites
+	ListSites       bool
+	AddSite         string
+	RemoveSite      string
+	SetSitePriority string
+	Search          bool
+
+	Filter string
+}
+
+var opts Options
+
+// print only if verbose is on
+func vPrintf(s string, x ...interface{}) {
+	if opts.Verbose {
+		fmt.Printf(s, x...)
+	}
+}
+
+func main() {
+	if opts.Help {
+		optarg.Usage()
+		return
+	}
+	if opts.Version {
+		fmt.Printf("%s\n", VERSION)
+		return
+	}
+	if opts.Verbose {
+		fmt.Printf("Verbosity on.\n")
+	}
+
+	if opts.ListSites {
+		ListSites()
+		return
+	}
+
+	if len(opts.AddSite) > 0 {
+		var priority int
+		var err error
+		name := opts.AddSite
+		if len(optarg.Remainder) == 0 {
+			fmt.Printf("Error: --add-site requires at least a name and a url for argument.")
+			return
+		}
+		url := optarg.Remainder[0]
+
+		// set the optional priority, or use the max
+		if len(optarg.Remainder) > 1 {
+			priority, err = strconv.Atoi(optarg.Remainder[1])
+			if err != nil {
+				fmt.Printf("Error with priority argument: %s\n", err)
+			}
+		} else {
+			db := config.Open()
+			defer db.Close()
+			row := db.QueryRow("select max(priority) from sites")
+			err = row.Scan(&priority)
+			if err != nil {
+				priority = 1
+			} else {
+				priority += 1
+			}
+		}
+
+		AddSite(name, url, priority)
+		return
+	}
+
+	if len(opts.RemoveSite) > 0 {
+		RemoveSite(opts.RemoveSite)
+		return
+	}
+
+	if opts.Update {
+		Update()
+	}
+
+	if opts.Search {
+		Search(optarg.Remainder...)
+	}
+
+}
+
+func Update() {
+	UpdateSites(false)
+}
+
+func ListSites() {
+	db := config.Open()
+	defer db.Close()
+
+	rows, err := db.Query("SELECT name, url, priority FROM sites ORDER BY priority")
+	if err != nil {
+		fmt.Printf("Error fetching sites: %s\n", err)
+	}
+	for rows.Next() {
+		site := new(Site)
+		err = rows.Scan(&site.Name, &site.Url, &site.Priority)
+		if err == nil {
+			fmt.Printf(" %d. %s [%s]\n", site.Priority, site.Name, site.Url)
+		} else {
+			fmt.Printf("Error: %s\n", err)
+		}
+	}
+}
+
+func Search(terms ...string) {
+	UpdateSites(false)
+
+	db := config.Open()
+	defer db.Close()
+	term := fmt.Sprintf("%%%s%%", strings.Join(terms, "%"))
+	q := "select name, key, url, site, updated from series where name like ?"
+	rows, err := db.Query(q, term)
+	if err != nil {
+		fmt.Printf("Error searching: %s\n", err)
+		return
+	}
+
+	for rows.Next() {
+		s := new(Series)
+		err = rows.Scan(&s.Name, &s.Key, &s.Url, &s.Site, &s.Updated)
+		if err != nil {
+			fmt.Printf("Error: %s\n", err)
+		}
+		fmt.Printf(" * %s (%s)\n", s.Name, s.Site)
+	}
+}
+
+func AddSite(name, url string, priority int) {
+	err := config.AddSite(name, url, priority)
+	if err != nil {
+		fmt.Printf("Error adding site: %s\n", err)
+	}
+}
+
+func RemoveSite(name string) {
+	err := config.RemoveSite(name)
+	if err != nil {
+		fmt.Printf("Error removing site: %s\n", err)
+	}
+}
+
+func init() {
+	optarg.HeaderFmt = "\n  %s options"
+
+	optarg.Add("h", "help", "Show help.", false)
+	optarg.Add("", "version", "Show version and exit.", false)
+	optarg.Add("v", "verbose", "Show more output.", false)
+
+	optarg.Header("Downloading")
+	optarg.Add("u", "update", "Update all site & series info.", false)
+	optarg.Add("", "sync", "Sync series info with what is on disk.", false)
+	optarg.Add("d", "download", "Download new chapters from series.", false)
+
+	optarg.Header("Sites")
+	optarg.Add("", "sites", "List sites.", false)
+	optarg.Add("", "add-site", "<name> <url> [priority], Add a site.", "")
+	optarg.Add("", "rm-site", "<name>, Remove a site.", "")
+	optarg.Add("", "set-site-priority", "<name> <priority>, Set download priority.", "")
+
+	optarg.Header("Series")
+	optarg.Add("l", "list", "List series.", false)
+	optarg.Add("s", "show", "List series.", false)
+	optarg.Add("a", "add-series", "Add a series.", false)
+	optarg.Add("r", "rm-series", "Remove a series.", false)
+	optarg.Add("f", "find", "Find a series.", false)
+	optarg.Add("", "search", "Alias for --find.", false)
+
+	for opt := range optarg.Parse() {
+		switch opt.Name {
+		case "help":
+			opts.Help = opt.Bool()
+		case "update":
+			opts.Update = opt.Bool()
+		case "version":
+			opts.Version = opt.Bool()
+		case "verbose":
+			opts.Verbose = opt.Bool()
+		// sites
+		case "sites":
+			opts.ListSites = opt.Bool()
+		case "add-site":
+			opts.AddSite = opt.String()
+		case "rm-site":
+			opts.RemoveSite = opt.String()
+		case "find", "search":
+			opts.Search = opt.Bool()
+		}
+	}
+
+}
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"time"
+)
+
+// update once per day unless forced
+var SITE_UPDATE_FREQUENCY = int64(86400) * 7
+var MAX_CONCURRENT_WORKERS = 3
+
+var client = &http.Client{
+	// keep user-agent:
+	// https://groups.google.com/forum/?fromgroups#!topic/golang-nuts/OwGvopYXpwE%5B1-25%5D
+	CheckRedirect: func(req *http.Request, via []*http.Request) error {
+		old := via[0]
+		req.Header.Set("User-Agent", old.UserAgent())
+		return nil
+	},
+}
+
+func toDate(timestamp int64) string {
+	if timestamp == 0 {
+		return "never"
+	}
+	t := time.Unix(timestamp, 0)
+	return t.Format(time.UnixDate)
+}
+
+func UpdateSites(force bool) {
+	var sites []Site
+	now := time.Now().Unix()
+	after := now
+	if !force {
+		after -= SITE_UPDATE_FREQUENCY
+	}
+
+	q := "select name, url, priority, updated from sites WHERE updated < ? ORDER BY priority"
+	db := config.Open()
+	rows, err := db.Query(q, after)
+	if err != nil {
+		panic(err)
+	}
+
+	for rows.Next() {
+		site := new(Site)
+		err = rows.Scan(&site.Name, &site.Url, &site.Priority, &site.Updated)
+		if err != nil {
+			fmt.Printf("Error: %s\n", err)
+		}
+		sites = append(sites, *site)
+	}
+
+	if len(sites) == 0 {
+		return
+	}
+
+	if !force {
+		fmt.Printf("Updating %d sites last updated over 1 week ago:\n", len(sites))
+	} else {
+		fmt.Printf("Force-updating %d sites:\n", len(sites))
+	}
+
+	sem := make(chan bool, MAX_CONCURRENT_WORKERS)
+	results := make([]map[string]string, 0)
+	cat := func(a, b []map[string]string) []map[string]string {
+		r := make([]map[string]string, len(a)+len(b))
+		copy(r, a)
+		copy(r[len(a):], b)
+		return r
+	}
+
+	for _, s := range sites {
+		sem <- true
+		go func(site Site) {
+			defer func() { <-sem }()
+			ret := UpdateSite(site)
+			results = cat(results, ret)
+		}(s)
+	}
+	for i := 0; i < cap(sem); i++ {
+		sem <- true
+	}
+
+	tx, err := db.Begin()
+	if err != nil {
+		panic(err)
+	}
+
+	fmt.Printf("Received %d total results\n", len(results))
+	q = "insert or replace into series (name, key, url, site, updated) values (?, ?, ?, ?, coalesce((select updated from series where url = ?), 0))"
+
+	for _, r := range results {
+		tx.Exec(q, r["name"], r["key"], r["url"], r["site"], r["url"])
+	}
+	for _, s := range sites {
+		tx.Exec("update sites set updated = ? where name = ?", now, s.Name)
+	}
+	tx.Commit()
+}
+
+func PrintDict(dict map[string]string) {
+	fmt.Printf("{\n")
+	for key, val := range dict {
+		fmt.Printf("  \"%s\": \"%s\",\n", key, val)
+	}
+	fmt.Printf("}\n")
+}
+
+func UrlJoin(strs ...string) string {
+	ss := make([]string, len(strs))
+	for i, s := range strs {
+		if i == 0 {
+			ss[i] = strings.TrimRight(s, "/")
+		} else {
+			ss[i] = strings.TrimLeft(s, "/")
+		}
+	}
+	return strings.Join(ss, "/")
+}
+
+var UpdateUrls = map[string]string{
+	"manga-access": "/manga/list",
+	"mangahere":    "/mangalist/",
+	"mangareader":  "/alphabetical",
+	"mangafox":     "/manga/",
+}
+
+func UpdateMangaaccess(site *Site, doc *Document) []map[string]string {
+	series := doc.CssSelect("#inner_page >div a")
+	data := make([]map[string]string, len(series))
+	for i, anchor := range series {
+		data[i] = map[string]string{
+			"site": site.Name,
+			"name": strings.Trim(anchor.Text(), "\t "),
+			"url":  UrlJoin(site.Url, anchor.Attr("href")),
+		}
+		spl := strings.Split(data[i]["url"], "/")
+		data[i]["key"] = spl[len(spl)-1]
+		data[i]["url"] = data[i]["url"] + "?mature_confirm=1"
+	}
+
+	fmt.Printf("Found %d series for manga-access\n", len(data))
+	PrintDict(data[0])
+	return data
+}
+
+func UpdateMangahere(site *Site, doc *Document) []map[string]string {
+	series := doc.CssSelect("div.list_manga li a")
+	data := make([]map[string]string, len(series))
+
+	for i, anchor := range series {
+		data[i] = map[string]string{
+			"site": site.Name,
+			"name": strings.Trim(anchor.Text(), " \t"),
+			"url":  strings.Trim(anchor.Attr("href"), " \t"),
+		}
+		if !strings.HasPrefix(data[i]["url"], "http") {
+			data[i]["url"] = UrlJoin(site.Url, data[i]["url"])
+		}
+		url := strings.TrimRight(data[i]["url"], "/")
+		spl := strings.Split(url, "/")
+		data[i]["key"] = spl[len(spl)-1]
+	}
+	fmt.Printf("Found %d series for mangahere\n", len(data))
+	PrintDict(data[0])
+	return data
+}
+
+func UpdateMangareader(site *Site, doc *Document) []map[string]string {
+	fmt.Printf("Update mangareader\n")
+	results := make([]map[string]string, 0)
+	return results
+}
+
+func UpdateMangafox(site *Site, doc *Document) []map[string]string {
+	fmt.Printf("Update mangafox\n")
+	results := make([]map[string]string, 0)
+	return results
+}
+
+var UpdateFunctions = map[string]func(*Site, *Document) []map[string]string{
+	"manga-access": UpdateMangaaccess,
+	"mangahere":    UpdateMangahere,
+	"mangareader":  UpdateMangareader,
+	"mangafox":     UpdateMangafox,
+}
+
+func UpdateSite(site Site) []map[string]string {
+	path, ok := UpdateUrls[site.Name]
+	updater := UpdateFunctions[site.Name]
+	none := make([]map[string]string, 0)
+
+	if !ok {
+		fmt.Printf("Unknown site-name %s, skipping update.\n", site.Name)
+		return none
+	}
+	url := UrlJoin(site.Url, path)
+	fmt.Printf("Updating %s via %s\n", site.Name, url)
+	var body []byte
+
+	// TESTING, REMOVE LATER
+
+	cachefile := site.Name + ".html"
+	if exists(cachefile) {
+		fmt.Printf("Reading in cached body in %s\n", cachefile)
+		body, _ = ioutil.ReadFile(cachefile)
+	} else {
+		req, err := http.NewRequest("GET", url, nil)
+		if err != nil {
+			fmt.Printf("Error creating request: %s\n", err)
+			return none
+		}
+
+		resp, err := client.Do(req)
+		if err != nil {
+			fmt.Printf("Error executing request: %s\n", err)
+			return none
+		}
+		defer resp.Body.Close()
+		body, err = ioutil.ReadAll(resp.Body)
+
+		if err != nil {
+			fmt.Printf("Error reading response: %s\n", err)
+			return none
+		}
+
+		// TESTING, REMOVE LATER
+		err = ioutil.WriteFile(site.Name+".html", body, 0655)
+		if err != nil {
+			fmt.Printf("Error writing out file %s.html\n", site.Name)
+		}
+	}
+
+	document := new(Document)
+	err := document.FromBytes(body)
+	if err != nil {
+		fmt.Printf("Error parsing document: %s\n", err)
+		return none
+	}
+	defer document.Free()
+	return updater(&site, document)
+}