6 월 27 일 실습 보고

     ,            ,      。     ,                   ：      a、b、c，a       ，b、c               ，  b     gridfs                         ，         b        。     md5   ，     gridfs   ，    。

package spider_handler

import (
	"constants"
	"database"
	"model"
	"utils"
	"fmt"
	"github.com/apex/log"
	"github.com/globalsign/mgo/bson"
	"github.com/satori/go.uuid"
	"github.com/spf13/viper"
	"io"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"runtime/debug"
	"strings"
	"sync"
)

const (
	Md5File = "md5.txt"
)

type SpiderSync struct {
	Spider model.Spider
}

func (s *SpiderSync) CreateMd5File(md5 string) {
	path := filepath.Join(viper.GetString("spider.path"), s.Spider.Name)
	utils.CreateDirPath(path)

	fileName := filepath.Join(path, Md5File)
	file := utils.OpenFile(fileName)
	defer utils.Close(file)
	if file != nil {
		if _, err := file.WriteString(md5 + "
"); err != nil {
			log.Errorf("file write string error: %s", err.Error())
			debug.PrintStack()
		}
	}
}

func (s *SpiderSync) CheckIsScrapy() {//         ,           scrapy.cfg   ,            .
	if s.Spider.Type == constants.Configurable {
		return
	}
	
	s.Spider.IsScrapy = utils.Exists(path.Join(s.Spider.Src, "scrapy.cfg"))
	if s.Spider.IsScrapy {
		s.Spider.Cmd = "scrapy crawl"
	}
	if err := s.Spider.Save(); err != nil {
		log.Errorf(err.Error())
		debug.PrintStack()
		return
	}
}

func (s *SpiderSync) AfterRemoveDownCreate() {
	if model.IsMaster() {
		s.CheckIsScrapy()
	}
}

func (s *SpiderSync) RemoveDownCreate(md5 string) {
	s.RemoveSpiderFile()
	s.Download()
	s.CreateMd5File(md5)
	s.AfterRemoveDownCreate()
}

//       
func (s *SpiderSync) RemoveSpiderFile() {
	path := filepath.Join(
		viper.GetString("spider.path"),
		s.Spider.Name,
	)
	//       ，       
	if err := os.RemoveAll(path); err != nil {
		log.Errorf("remove spider files error: %s, path: %s", err.Error(), path)
		debug.PrintStack()
	}
}


//     
func (s *SpiderSync) Download() {
	spiderId := s.Spider.Id.Hex()
	fileId := s.Spider.FileId.Hex()
	isDownloading, key := s.CheckDownLoading(spiderId, fileId)
	if isDownloading {
		log.Infof(fmt.Sprintf("spider is already being downloaded, spider id: %s", s.Spider.Id.Hex()))
		return
	} else {
		_ = database.RedisClient.HSet("spider", key, key)
	}

	session, gf := database.GetGridFs("files")
	defer session.Close()

	f, err := gf.OpenId(bson.ObjectIdHex(fileId))
	defer utils.Close(f)
	if err != nil {
		log.Errorf("open file id: " + fileId + ", spider id:" + spiderId + ", error: " + err.Error())
		debug.PrintStack()
		return
	}

	//     ID
	randomId := uuid.NewV4()
	tmpPath := viper.GetString("other.tmppath")
	if !utils.Exists(tmpPath) {
		if err := os.MkdirAll(tmpPath, 0777); err != nil {
			log.Errorf("mkdir other.tmppath error: %v", err.Error())
			return
		}
	}
	//       
	tmpFilePath := filepath.Join(tmpPath, randomId.String()+".zip")
	tmpFile := utils.OpenFile(tmpFilePath)

	//           
	if _, err := io.Copy(tmpFile, f); err != nil {
		log.Errorf("copy file error: %s, file_id: %s", err.Error(), f.Id())
		debug.PrintStack()
		return
	}

	//              
	dstPath := filepath.Join(
		viper.GetString("spider.path"),
		s.Spider.Name,
	)
	if err := utils.DeCompress(tmpFile, dstPath); err != nil {
		log.Errorf(err.Error())
		debug.PrintStack()
		return
	}

	cmd := exec.Command("chmod", "-R", "777", dstPath)
	if err := cmd.Run(); err != nil {
		log.Errorf(err.Error())
		debug.PrintStack()
		return
	}

	//       
	if err := tmpFile.Close(); err != nil {
		log.Errorf(err.Error())
		debug.PrintStack()
		return
	}

	//       
	if err := os.Remove(tmpFilePath); err != nil {
		log.Errorf(err.Error())
		debug.PrintStack()
		return
	}

	_ = database.RedisClient.HDel("spider", key)
}

              ，   file    ，    children     ，           。

package services

import (
	"model"
	"github.com/apex/log"
	"os"
	"path"
	"runtime/debug"
	"strings"
)

func GetFileNodeTree(dstPath string, level int) (f model.File, err error) {
	return getFileNodeTree(dstPath, level, dstPath)
}

func getFileNodeTree(dstPath string, level int, rootPath string) (f model.File, err error) {
	dstF, err := os.Open(dstPath)
	if err != nil {
		log.Errorf(err.Error())
		debug.PrintStack()
		return f, err
	}
	defer dstF.Close()
	fileInfo, err := dstF.Stat()
	if err != nil {
		log.Errorf(err.Error())
		debug.PrintStack()
		return f, nil
	}
	if !fileInfo.IsDir() { //  dstF   
		return model.File{
			Label:    fileInfo.Name(),
			Name:     fileInfo.Name(),
			Path:     strings.Replace(dstPath, rootPath, "", -1),
			IsDir:    false,
			Size:     fileInfo.Size(),
			Children: nil,
		}, nil
	} else { //  dstF    
		dir, err := dstF.Readdir(0) //               fileInfo
		if err != nil {
			log.Errorf(err.Error())
			debug.PrintStack()
			return f, nil
		}
		f = model.File{
			Label:    path.Base(dstPath),
			Name:     path.Base(dstPath),
			Path:     strings.Replace(dstPath, rootPath, "", -1),
			IsDir:    true,
			Size:     0,
			Children: nil,
		}
		for _, subFileInfo := range dir {
			subFileNode, err := getFileNodeTree(path.Join(dstPath, subFileInfo.Name()), level+1, rootPath)
			if err != nil {
				log.Errorf(err.Error())
				debug.PrintStack()
				return f, err
			}
			f.Children = append(f.Children, subFileNode)
		}
		return f, nil
	}
}