golang和jieba实现的一个简易的全文搜索引擎

import (
    "encoding/xml"
    "os"
)

type Document struct {
    Title string
    URL   string
    Text  string
    ID    int
}

// LoadData 加载表单数据
func loadData(c *gin.Context) ([]Document, error) {
 var docs []Document
 // 查找所有文章
 posts, _, err := models.FindAllPosts()
 if err != nil {
  return nil, err
 }
 // 创建文档
 for i := range posts {
  doc := Document{
   Title: posts[i].Title,
   URL:   "",
   Text:  posts[i].PlainContent,
   ID:    int(posts[i].ID),
  }
  docs = append(docs, doc)
 }
 return docs, nil
}

import "github.com/yanyiwu/gojieba"

// analyze 使用jieba分词进行中文分词
func analyze(text string) []string {
 x := gojieba.NewJieba()
 defer x.Free()
 tokens := x.CutForSearch(text, true)
 return tokens
}

type index map[string][]int

// add adds documents to the index.
func (idx index) add(docs []Document) {
 for _, doc := range docs {
  for _, token := range analyze(doc.Text) {
   ids := idx[token]
   if ids != nil && ids[len(ids)-1] == doc.ID {
    continue
   }
   idx[token] = append(ids, doc.ID)
  }
 }
}

// intersection 获取a、b数据的交集
// a and b 需要升序排序并且不包含重复
func intersection(a []int, b []int) []int {
 maxLen := len(a)
 if len(b) > maxLen {
  maxLen = len(b)
 }
 r := make([]int, 0, maxLen)
 var i, j int
 for i < len(a) && j < len(b) {
  if a[i] < b[j] {
   i++
  } else if a[i] > b[j] {
   j++
  } else {
   r = append(r, a[i])
   i++
   j++
  }
 }
 return r
}

// search queries the index for the given text.
func (idx index) search(text string) []int {
 var r []int
 for _, token := range analyze(text) {
  if ids, ok := idx[token]; ok {
   if r == nil {
    r = ids
   } else {
    r = intersection(r, ids)
   }
  } else {
   // Token doesn't exist.
   return nil
  }
 }
 return r
}

// Search 搜索
func Search(c *gin.Context) {
 s := "HTTPS 和 HTTP/2"
 // 加载文章
 docs, _ := loadData(c)
 // 创建索引
 idx := make(index)
 // 根据数据添加索引
 idx.add(docs)
 // 查找文本匹配索引id
 matchedIDs := idx.search(s)
 start := time.Now()
 log.Printf("Search found %d documents in %v", len(matchedIDs), time.Since(start))
}