Debian-facile

package main

import (
  "fmt"
  "io"
  "io/ioutil"
  "log"
  "net/http"
  "os"
  "os/exec"
  "regexp"
)

var (
  url          = "https://debian-facile.org/img/avatars"
  outputFolder = "images-go"
)

func download(content []byte) (string, error) {
  nameByte, err := retriveImageName(content)
  imageName := string(nameByte)

  if err != nil {
    return "", fmt.Errorf("%s", err.Error())
  }

  file, err := os.Create(fmt.Sprintf("%s/%s", outputFolder, imageName))

  if err != nil {
    return "", fmt.Errorf("Unable to create file: %s", err.Error())
  }

  defer file.Close()

  endPoint := fmt.Sprintf("%s/%s", url, imageName)
  resp, err := http.Get(endPoint)

  if err != nil {
    return "", fmt.Errorf("Unable to get image: %s", err.Error())
  }

  defer resp.Body.Close()

  _, err = io.Copy(file, resp.Body)

  if err != nil {
    return "", fmt.Errorf("Unable to write downloaded content into filesystem: %s", err.Error())
  }

  return fmt.Sprintf("%s downloaded successfully from %s", imageName, endPoint), nil
}

func retriveImageName(b []byte) (string, error) {
  imgRegexp, err := regexp.Compile(`\"(.*?)\"`)

  if err != nil {
    return "", err
  }

  found := imgRegexp.Find(b)

  ok, err := regexp.Match(`.png|.gif|.jpg|jpeg`, found)

  if err != nil || !ok {
    return "", fmt.Errorf("%s is not a valid image source", string(found))
  }

  name := string(found)
  name = name[:len(name)-1]
  name = name[1:]

  return name, nil
}

func crawl(body []byte) {
  hrefRegexp, err := regexp.Compile(`href="(.*?)\"`)

  if err != nil {
    log.Printf("%s\n", err.Error())
  }

  hrefBytes := hrefRegexp.FindAll(body, -1)

  ch := make(chan string)

  defer close(ch)

  for _, href := range hrefBytes {
    go func(h []byte) {
      downloaded, err := download(h)

      if err != nil {
        ch <- err.Error()
      } else {
        ch <- downloaded
      }
    }(href)
  }

  for i := 0; i < len(hrefBytes); i++ {
    select {
    case image := <-ch:
      log.Println(image)
    }
  }
}

func main() {
  res, err := http.Get(url)

  if err != nil {
    panic(fmt.Sprintf("Unable to fetch debian facile page: %s", err.Error()))
  }

  defer res.Body.Close()

  bodyBytes, err := ioutil.ReadAll(res.Body)

  if err != nil {
    panic(fmt.Sprintf("Unable to read request body: %s", err.Error()))
  }

  crawl(bodyBytes)
  cmd := exec.Command("montage", outputFolder+"/*.jpg", outputFolder+"/*png", outputFolder+"/-title", "trombi", "trombi-go.png")
  err = cmd.Run()
  if err != nil {
    log.Fatal(err)
  }
}
 

package main

import (
  "gitlab.com/df-trombi/crawler"
)

var (
  url          = "https://debian-facile.org/img/avatars"
  outputFolder = "images-go"
)

func main() {
  crawler := crawler.New().SetURL(url).SetOutputDirectory("images-go").SetFormats(".png", ".jpg", ".jpeg", ".gif")
  crawler.Crawl()
}
 

package helper

import (
  "io/ioutil"
  "net/http"
)

// GetResponseBody open, get and close request body
func GetResponseBody(response *http.Response) ([]byte, error) {
  bodyReader := response.Body

  defer response.Body.Close()

  bodyBytes, err := ioutil.ReadAll(bodyReader)

  if err != nil {
    return nil, err
  }

  return bodyBytes, nil
}
 

package crawler

import (
  "fmt"
  "io"
  "log"
  "net/http"
  "os"
  "os/exec"
  "regexp"

  "gitlab.com/df-trombi/helper"
)

func (c *Crawler) download(content []byte) (string, error) {
  nameByte, err := c.retriveImageName(content)
  imageName := string(nameByte)

  if err != nil {
    return "", fmt.Errorf("%s", err.Error())
  }

  file, err := os.Create(fmt.Sprintf("%s/%s", c.outputDirectory, imageName))

  if err != nil {
    return "", fmt.Errorf("Unable to create file: %s", err.Error())
  }

  defer file.Close()

  endPoint := fmt.Sprintf("%s/%s", c.url, imageName)
  resp, err := http.Get(endPoint)

  if err != nil {
    return "", fmt.Errorf("Unable to get image: %s", err.Error())
  }

  defer resp.Body.Close()

  _, err = io.Copy(file, resp.Body)

  if err != nil {
    return "", fmt.Errorf("Unable to write downloaded content into filesystem: %s", err.Error())
  }

  return fmt.Sprintf("%s downloaded successfully from %s", imageName, endPoint), nil
}

func (c *Crawler) retriveImageName(b []byte) (string, error) {
  imgRegexp, err := regexp.Compile(`\"(.*?)\"`)

  if err != nil {
    return "", err
  }

  found := imgRegexp.Find(b)

  ok, err := regexp.Match(`.png|.gif|.jpg|jpeg`, found)

  if err != nil || !ok {
    return "", fmt.Errorf("%s is not a valid image source", string(found))
  }

  name := string(found)
  name = name[:len(name)-1]
  name = name[1:]

  return name, nil
}

func (c *Crawler) crawl(body []byte) {
  hrefRegexp, err := regexp.Compile(`href="(.*?)\"`)

  if err != nil {
    log.Printf("%s\n", err.Error())
  }

  hrefBytes := hrefRegexp.FindAll(body, -1)

  ch := make(chan string)

  defer close(ch)

  for _, href := range hrefBytes {
    go func(h []byte) {
      downloaded, err := c.download(h)

      if err != nil {
        ch <- err.Error()
      } else {
        ch <- downloaded
      }
    }(href)
  }

  for i := 0; i < len(hrefBytes); i++ {
    select {
    case image := <-ch:
      log.Println(image)
    }
  }
}

// Crawler struct
type Crawler struct {
  url             string
  outputDirectory string
  formats         []string
}

// SetFormats fluent format setter for Crawler
func (c *Crawler) SetFormats(formats ...string) *Crawler {
  c.formats = formats
  return c
}

// GetFormats formats getter for Crawler
func (c *Crawler) GetFormats() []string {
  return c.formats
}

// SetURL fluent url setter for Crawler
func (c *Crawler) SetURL(url string) *Crawler {
  c.url = url
  return c
}

// SetOutputDirectory fluent outputDirectory setter for Crawler
func (c *Crawler) SetOutputDirectory(directory string) *Crawler {
  c.outputDirectory = directory
  return c
}

// GetOutputDirectory outputDirectory getter for Crawler
func (c *Crawler) GetOutputDirectory() string {
  return c.outputDirectory
}

// GetURL url getter for Crawler
func (c *Crawler) GetURL() string {
  return c.url
}

// New get new Crawler
func New() *Crawler {
  return &Crawler{}
}

func (c *Crawler) buildOutputFilePath() []string {
  var res []string
  for _, format := range c.formats {
    res = append(res, fmt.Sprintf("%s/*%s", c.outputDirectory, format))
  }
  return res
}

func (c *Crawler) buildCommand() *exec.Cmd {
  arg := c.buildOutputFilePath()
  arg = append(arg, "-title")
  arg = append(arg, "trombi")
  arg = append(arg, "trombi-go.png")
  return exec.Command("montage", arg...)
}

// Crawl url
func (c *Crawler) Crawl() {
  // get pointeur of Response object
  res, err := http.Get(c.url)

  if err != nil {
    panic(fmt.Sprintf("Unable to fetch debian facile page: %s", err.Error()))
  }

  // open, get and close response body
  bodyBytes, err := helper.GetResponseBody(res)

  if err != nil {
    panic(fmt.Sprintf("Unable to read request body: %s", err.Error()))
  }

  c.crawl(bodyBytes)

  cmd := c.buildCommand()
  err = cmd.Run()

  if err != nil {
    log.Fatal(err)
  }
}