1. Jason Moiron
  2. ukiyo

Commits

Jason Moiron  committed 06d6517

chapter downloading on the two initially supported sites, show download path in help

Comments (0)

Files changed (6)

File config.go Modified

View file
  • Ignore whitespace
  • Hide word diff
 }
 
 func (c *Config) SetVal(key, val string) {
-	Exec(c.db, "update config set val=? where key=?", val, key)
+	Exec(c.db, "update config set value=? where key=?", val, key)
 }
 
 func (c *Config) init() {
 	if err != nil {
 		fmt.Errorf("Could not read key 'DownloadPath' from config: %q\n", err)
 	}
+
+	rows := Query(c.db, "select name from sites order by priority")
+	for rows.Next() {
+		var s string
+		rows.Scan(&s)
+		c.SiteOrder = append(c.SiteOrder, s)
+	}
+
 }
 
 // Abstract interface covering Exec & Query so that transactions and the db can

File files.go Modified

View file
  • Ignore whitespace
  • Hide word diff
 			continue
 		}
 		realpath := filepath.Join(path, file.Name())
-		zippath := filepath.Join(filepath.Dir(path), file.Name())
+		zippath := filepath.Join(filepath.Base(path), file.Name())
 		f, err := writer.Create(zippath)
 		if err != nil {
 			return err

File http.go Modified

View file
  • Ignore whitespace
  • Hide word diff
 }
 
 // Download a url to a path
-func HttpDownloadTo(url, path string)
+func HttpDownloadTo(url, path string) error {
+	body, err := HttpGet(url)
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(path, body, 0644)
+	if err != nil {
+		return err
+	}
+	return nil
+}
 
 // return the base url for a given url string
 func BaseUrl(u string) string {

File parse.go Modified

View file
  • Ignore whitespace
  • Hide word diff
 	"github.com/moovweb/gokogiri/xml"
 	"github.com/moovweb/gokogiri/xpath"
 	"regexp"
+	"strings"
 	"time"
 	"unsafe"
 )
 	return t.Format(time.UnixDate)
 }
 
+func FileExtension(str string) string {
+	spl := strings.Split(str, ".")
+	return spl[len(spl)-1]
+}
+
 func tick() { fmt.Printf("%s\n", time.Now().String()) }
 
 // Selectable implements a simple interface which allows to get the inner text

File ukiyo.go Modified

View file
  • Ignore whitespace
  • Hide word diff
 		vPrintf("Verbosity on.\n")
 	}
 
+	if len(opts.SetDownloadPath) > 0 {
+		config.SetVal("DownloadPath", opts.SetDownloadPath)
+		return
+	}
+
 	if len(opts.SetSitePriority) > 0 {
 		if len(optarg.Remainder) != 1 {
 			fmt.Printf("Error: --set-site-priority requires a name and a priority.\n")
 
 	if opts.Update {
 		Update()
+		return
 	}
 
 	if opts.Search {
 		Search(optarg.Remainder...)
+		return
 	}
 
 	if opts.Show {
 		Show(optarg.Remainder...)
+		return
 	}
 
+	DownloadChapters(optarg.Remainder...)
 }
 
 func Update() {
 	}
 }
 
+func DownloadChapters(terms ...string) {
+	UpdateSites()
+	series, err := FindOneSeries(true, terms...)
+	if err != nil {
+		fmt.Printf("Error searching for terms: %s\n", err)
+	}
+
+	if len(series) == 0 {
+		fmt.Printf("No series found matching term %s\n", strings.Join(terms, " "))
+		return
+	}
+	if len(series) > 1 {
+		fmt.Printf("Found %d series matching term \"%s\":\n", len(series), strings.Join(terms, " "))
+		for _, s := range series {
+			fmt.Printf(" * %s (%s)\n", s.Name, s.Site)
+		}
+		fmt.Printf("Try using the exact name (or key) of a series.\n")
+		return
+	}
+
+	chapters := UpdateChapters(series[0].Name)
+	for _, c := range chapters {
+		if opts.Filter.Match(c.Number) {
+			DownloadChapter(c)
+		}
+	}
+
+}
+
 func AddSite(name, url string, priority int) {
 	err := config.AddSite(name, url, priority)
 	if err != nil {
 	optarg.Add("u", "update", "Update all site & series info.", false)
 	optarg.Add("", "sync", "Sync series info with what is on disk.", false)
 	optarg.Add("d", "download", "Download new chapters from series.", false)
-	optarg.Add("", "set-download-path", "Change destination for sync and downloads.", "")
+	optarg.Add("", "set-download-path", fmt.Sprintf("Change destination for sync and downloads. (Current: %s)", config.DownloadPath), "")
 
 	optarg.Header("Sites")
 	optarg.Add("", "sites", "List sites.", false)

File update.go Modified

View file
  • Ignore whitespace
  • Hide word diff
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
 	"strings"
 	"time"
 )
 	"mangareader":  UpdateChaptersMangareader,
 	"mangafox":     UpdateChaptersMangafox,
 }
+
+func SelectUrl(chapter *Chapter) (string, string) {
+	spl := strings.Split(chapter.Site, ",")
+	m := make(map[string]int, len(spl))
+	for i, s := range spl {
+		m[s] = i
+	}
+	for _, s := range config.SiteOrder {
+		_, ok := m[s]
+		if ok {
+			spl = strings.Split(chapter.Url, ",")
+			return s, spl[m[s]]
+		}
+	}
+	return "", ""
+}
+
+// TODO: do everything through this, as it's superior to what i've been doing
+
+type Downloader interface {
+	// get page urls via a document and the url string of that document
+	GetPageUrls(string, *Document) []string
+	GetImageUrl(*Document) string
+}
+
+type MangaAccess struct{}
+
+func (m *MangaAccess) GetPageUrls(url string, doc *Document) []string {
+	options := doc.CssSelect("select#page_switch option")
+	if len(options) < 1 {
+		return []string{}
+	}
+	urls := make([]string, len(options)-1)
+	base := BaseUrl(url)
+
+	for i, o := range options[1:] {
+		urls[i] = UrlJoin(base, o.Attr("value"))
+	}
+	return urls
+}
+
+func (m *MangaAccess) GetImageUrl(doc *Document) string {
+	img := doc.CssSelect("div#pic img")
+	if len(img) == 0 {
+		return ""
+	}
+	return img[0].Attr("src")
+}
+
+type MangaHere struct{}
+
+func (m *MangaHere) GetPageUrls(url string, doc *Document) []string {
+	options := doc.CssSelect(".readpage_top .go_page option")
+	if len(options) < 1 {
+		return []string{}
+	}
+	urls := make([]string, len(options)-1)
+	base := BaseUrl(url)
+
+	for i, o := range options[1:] {
+		fragment := o.Attr("value")
+		if strings.HasPrefix(fragment, "http") {
+			urls[i] = fragment
+		} else {
+			urls[i] = UrlJoin(base, fragment)
+		}
+	}
+	return urls
+}
+
+func (m *MangaHere) GetImageUrl(doc *Document) string {
+	img := doc.CssSelect("#image")
+	if len(img) == 0 {
+		return ""
+	}
+	return img[0].Attr("src")
+}
+
+var Downloaders = map[string]Downloader{
+	"manga-access": new(MangaAccess),
+	"mangahere":    new(MangaHere),
+}
+
+func DownloadChapter(chapter *Chapter) error {
+	site, url := SelectUrl(chapter)
+	// fmt.Printf(" %s %s (%s, %s)\n", chapter.Series, chapter.Number, site, url)
+	downloader := Downloaders[site]
+
+	doc, err := HttpGetDocument(url)
+	if err != nil {
+		return err
+	}
+	var destzip string
+	destpath := filepath.Join(config.DownloadPath, chapter.Series, chapter.Number)
+	if chapter.Numberf != 0 {
+		destzip = filepath.Join(config.DownloadPath, chapter.Series,
+			fmt.Sprintf("%s-c%03d.zip", chapter.Series, int(chapter.Numberf)))
+	} else {
+		destzip = filepath.Join(config.DownloadPath, chapter.Series,
+			fmt.Sprintf("%s-c%s.zip", chapter.Series, chapter.Number))
+	}
+
+	page_urls := downloader.GetPageUrls(url, doc)
+	numwidth := len(fmt.Sprintf("%d", len(page_urls)))
+	numfmt := fmt.Sprintf("%%0%dd", numwidth)
+	// fmt.Printf("Making destination dir: %s", destpath)
+	os.MkdirAll(destpath, 0755)
+	// fmt.Printf("Number format: %s\n", numfmt)
+
+	vPrintf("Page Urls: %v\n", page_urls)
+	update := fmt.Sprintf("Downloading %s %s (from %s): %%d of %%d", chapter.Series, chapter.Number, site)
+
+	type Img struct {
+		num int
+		url string
+	}
+
+	sem := make(chan bool, MAX_CONCURRENT_WORKERS)
+	images := make(chan Img, len(page_urls))
+	completed := make(chan int)
+	// send the first image on the images channel
+	images <- Img{1, downloader.GetImageUrl(doc)}
+
+	// print a little updater in place about pages we're loading
+	go func(max int) {
+		c := 0
+		fmt.Printf(update+"\r", c, max)
+		os.Stdout.Sync()
+		for _ = range completed {
+			c++
+			fmt.Printf(update+"\r", c, max)
+			os.Stdout.Sync()
+			if c == max {
+				fmt.Printf(update+"\n", c, max)
+				return
+			}
+		}
+	}(len(page_urls))
+
+	for i, s := range page_urls {
+		sem <- true
+		go func(num int, url string) {
+			defer func() { <-sem }()
+			doc, err := HttpGetDocument(url)
+			if err != nil {
+				fmt.Printf("Error fetching page %03d (%s)\n", num, url)
+				return
+			}
+			images <- Img{num, downloader.GetImageUrl(doc)}
+		}(i+2, s)
+	}
+
+	for i := 0; i < len(page_urls); i++ {
+		sem <- true
+		go func() {
+			defer func() { <-sem }()
+			img := <-images
+			if len(img.url) == 0 {
+				fmt.Printf("Error fetching page %03d (null url)\n", img.num)
+				return
+			}
+			ext := FileExtension(img.url)
+			name := fmt.Sprintf(numfmt+".%s", img.num, ext)
+			destfile := filepath.Join(destpath, name)
+			// fmt.Printf("#%d (%s) to %s\n", img.num, img.url, destfile)
+			HttpDownloadTo(img.url, destfile)
+			completed <- 1
+		}()
+	}
+	// wait for everything to finish...
+	for i := 0; i < cap(sem); i++ {
+		sem <- true
+	}
+
+	ZipDir(destpath, destzip)
+	os.RemoveAll(destpath)
+
+	return nil
+}