爬虫使用布隆过滤器

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70

package bloomfilter

import (
	"fmt"
	"github.com/steakknife/bloomfilter"
	"hash/crc32"
	"testing"
)

func Test_bloomFilterRead(t *testing.T) {
	_,_, err := bloomfilter.ReadFile("1.bf.gz") // read the BF to another var
	if err != nil {
		panic(err)
	}
	//
	//bf2.Add("http://www.baidu.com/my//1.php")
	//if bf2.Contains("http://www.baidu.com/my/1.php") {
	//	fmt.Println("contains...")
	//}
}
type stringhash string

func (h stringhash) Write([]byte) (int, error) {
	panic("Unimplemented")
}

func (h stringhash) Sum([]byte) []byte {
	panic("Unimplemented")
}

func (h stringhash) Reset() {
	panic("Unimplemented")
}

func (h stringhash) BlockSize() int {
	panic("Unimplemented")
}

func (h stringhash) Size() int {
	panic("Unimplemented")
}

func (h stringhash) Sum64() uint64 {
	v := int(crc32.ChecksumIEEE([]byte(h)))
	if v >= 0 {
		return uint64(v)
	}
	if -v >= 0 {
		return uint64(-v)
	}
	// v == MinInt
	return 0

}

func Test_bf2(t *testing.T) {
	bf, err := bloomfilter.NewOptimal(maxElements, probCollide)
	if err != nil {
		panic(err)
	}

	bf.Add(stringhash("http://lyr-2000.github.io/index.html"))
	bf.Add(stringhash("http://www.baidu.com/index.html"))
	fmt.Println(bf.Contains(stringhash("http://www.baidu.com/index.html")))
	fmt.Println(bf.Contains(stringhash("http://www.baidu.com/index2.jsp")))




}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65

package controller

import (
	"fmt"
	"github.com/gin-gonic/gin"
	"github.com/spf13/viper"
	"novel_server_v1/util/bloomfilter"
	"os"
	"os/signal"
	"syscall"
)



var onlyOneSignalHandler = make(chan struct{})
var shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM}
// SetupSignalHandler registered for SIGTERM and SIGINT. A stop channel is returned
// which is closed on one of these signals. If a second signal is caught, the program
// is terminated with exit code 1.
func SetupSignalHandler() (stopCh <-chan struct{}) {
	close(onlyOneSignalHandler) // panics when called twice

	stop := make(chan struct{})
	c := make(chan os.Signal, 2) //2表示chan的长度，输入多少次，就可以实现Control+c执行动作多少次
	signal.Notify(c, shutdownSignals...)
	go func() {
		<-c
		fmt.Println("正在退出...")
		bloomfilter.SaveFilter(bloomfilter.BF)
		close(stop)
		//<-c
		fmt.Println("程序已安全退出")
		os.Exit(1) // second signal. Exit directly.
	}()

	return stop
}

func CaptureCtrlC(stopCh <-chan struct{}){
	//捕获 ctrl +c 停止
	go func() {
		<-stopCh
	}()

}

func Start(r *gin.Engine) {
	CaptureCtrlC(SetupSignalHandler())



	UseNovelController(r)
	UseLoginController(r)
	UseScriptController(r)
	port := viper.GetString("port")
	fmt.Printf("port := %#v\n", port)
	if port == "" {
		panic("没有规定运行端口，panic")
	}
	err := r.Run(port)
	if err != nil {
		fmt.Printf("程序启动失败 := %#v\n", err)

	}
}