From 17897cc87cb6897a8ba597ef4b0f6a00e107c878 Mon Sep 17 00:00:00 2001 From: zcy <290198252@qq.com> Date: Sun, 7 Feb 2021 17:21:19 +0800 Subject: [PATCH] no message --- db/elasticEngine.go | 71 +++++++++++++++++++++++++++-- model/hardware.go | 75 +++++++++++++++--------------- model/port.go | 103 ++++++++++++++++++++---------------------- test/portData_test.go | 28 +++++++++--- 4 files changed, 174 insertions(+), 103 deletions(-) diff --git a/db/elasticEngine.go b/db/elasticEngine.go index c0e1f05..9a02cd8 100644 --- a/db/elasticEngine.go +++ b/db/elasticEngine.go @@ -13,7 +13,7 @@ import ( const ( ERROR_PTR = "null pointer error" - INPUT_TYPE_ERROR = "wrong input parameter" + INPUT_TYPE_ERROR = "wrong input parameter: " CREATED_ERROR = "create error" DELETE_ERROR = "delete error" INDEX_EXISTED = "index existed" @@ -60,7 +60,71 @@ func (p *ElkEngine) Delete(query elastic.Query, index string) error { return nil } -func (p *ElkEngine) Query(index string, query elastic.Query, typ reflect.Type, +func (p *ElkEngine) Query(index string, query elastic.Query, v interface{}, + limit int, offset int) ([]string, error) { + + if reflect.ValueOf(v).Kind() != reflect.Ptr { + return nil, errors.New(INPUT_TYPE_ERROR + "shoulbe be Ptr") + } + if reflect.ValueOf(v).Elem().Kind() != reflect.Slice { + return nil, errors.New(INPUT_TYPE_ERROR + "shoulbe be Slice") + } + if reflect.ValueOf(v).Elem().Type().Elem().Kind() != reflect.Struct { + return nil, errors.New(INPUT_TYPE_ERROR + "shoulbe be Struct") + } + eletype := reflect.ValueOf(v).Elem().Type().Elem() + obj := reflect.ValueOf(v).Elem() + objAdd := make([]reflect.Value, 0) + + if nil != p { + if limit == 0 { + res, err := p.cli.Search(index).Query(query).Do(context.Background()) + if err != nil { + print(err) + return nil, err + } + + id := []string{} + for _, vs := range res.Hits.Hits { + id = append(id, vs.Id) + } + return id, nil + } else { + res, err := p.cli.Search(index).Query(query).Size(limit).From(limit * offset).Do(context.Background()) + if err != nil { + print(err) + return nil, err + } + id := []string{} + for _, vs := range res.Hits.Hits { + id = append(id, vs.Id) + data, e := vs.Source.MarshalJSON() + if nil != e { + log.Print(e.Error()) + } + mapobj := map[string]interface{}{} + e = json.Unmarshal(data, &mapobj) + if nil != e { + log.Print(e.Error()) + } + obj, e := utils.UnmarshalJson2StructGen(eletype, mapobj) + log.Print(obj) + if nil != e { + log.Print(e.Error()) + } + objAdd = append(objAdd, reflect.ValueOf(obj)) + //v = append(v, obj) + } + addOp := reflect.Append(obj, objAdd...) + obj.Set(addOp) + return id, nil + } + } else { + return nil, errors.New(ERROR_PTR) + } +} + +func (p *ElkEngine) QueryGen(index string, query elastic.Query, typ reflect.Type, limit int, offset int) ([]interface{}, []string, error) { rets := []interface{}{} if nil != p { @@ -98,9 +162,6 @@ func (p *ElkEngine) Query(index string, query elastic.Query, typ reflect.Type, if nil != e { log.Print(e.Error()) } - // for k,_ := range mapobj{ - // value := reflect.ValueOf(obj) - // } rets = append(rets, obj) } return rets, id, nil diff --git a/model/hardware.go b/model/hardware.go index 11b9544..8f1c748 100644 --- a/model/hardware.go +++ b/model/hardware.go @@ -14,7 +14,7 @@ const ( ERR_COLUMN_EXISTED = "column_existed" ) -func HardwareTypeMapping() (string){ +func HardwareTypeMapping() string { return `"mappings":{ "hardware":{ "properties":{ @@ -30,75 +30,72 @@ func HardwareTypeMapping() (string){ // this api is based on elasticsearch type Hardware struct { - ID string `json:"_id,omitempty"` - BuyDate string `json:"buy_date,omitempty"` //购入时间 - Name string `json:"name,omitempty"` // 名字 - Desc string `json:"desc,omitempty"` // 描述 - Pic string `json:"pic,omitempty"` // 图片 - Doc string `json:"doc,omitempty"` //文档资料 + ID string `json:"_id,omitempty"` + BuyDate string `json:"buy_date,omitempty"` //购入时间 + Name string `json:"name,omitempty"` // 名字 + Desc string `json:"desc,omitempty"` // 描述 + Pic string `json:"pic,omitempty"` // 图片 + Doc string `json:"doc,omitempty"` //文档资料 } -func (this *Hardware )CreateHardware( ) error{ - if nil == this{ +func (this *Hardware) CreateHardware() error { + if nil == this { return errors.New(utils.ERRNULLPOINTER) } log.Print(this.Name) matchPhraseQuery := elastic.NewMatchQuery("name", this.Name) - existedHardware,e := QueryHardwares(matchPhraseQuery,10,0) - log.Print(e,existedHardware) + existedHardware, e := QueryHardwares(matchPhraseQuery, 10, 0) + log.Print(e, existedHardware) - for _,v := range existedHardware{ - if v.Name == this.Name{ + for _, v := range existedHardware { + if v.Name == this.Name { log.Print(v.ID) return errors.New(ERR_COLUMN_EXISTED) } } - e = db.GetElastic().Create("hardware_data","0","",*this) - if nil != e{ + e = db.GetElastic().Create("hardware_data", "0", "", *this) + if nil != e { log.Print("shit1") log.Print(e.Error()) return e } log.Print("shit2") - return nil; + return nil } -func GetHardwares(limit int,size int) ([]Hardware,error){ +func GetHardwares(limit int, size int) ([]Hardware, error) { var ret []Hardware - data,ids,e := db.GetElastic().Query("hardware_data",nil,reflect.TypeOf(Hardware{}),limit,size) - if nil != e{ - return nil,e + ids, e := db.GetElastic().Query("hardware_data", nil, reflect.TypeOf(Hardware{}), limit, size) + if nil != e { + return nil, e } i := 0 - for _,v := range data{ - ret = append(ret,v.(Hardware)) - ret[i].ID = ids[i] + for _, v := range ids { + ret[i].ID = v i++ } - return ret,nil + return ret, nil } -func QueryHardwares(query elastic.Query,limit int,offset int) ([]Hardware,error){ - var ret []Hardware - data,ids,e := db.GetElastic().Query("hardware_data",query,reflect.TypeOf(Hardware{}),limit,offset) - log.Print(data) - if nil != e{ - return nil,e +func QueryHardwares(query elastic.Query, limit int, offset int) ([]Hardware, error) { + ret := []Hardware{} + ids, e := db.GetElastic().Query("hardware_data", query, ret, limit, offset) + if nil != e { + return nil, e } i := 0 - for _,v := range data{ - ret = append(ret,v.(Hardware)) - ret[i].ID = ids[i] + for _, v := range ids { + ret[i].ID = v i++ } - return ret,nil + return ret, nil } -func DeleteHardware(name string) error{ - query := elastic.NewTermQuery("name",name) - err := db.GetElastic().Delete(query,"hardware_data") - if err != nil{ +func DeleteHardware(name string) error { + query := elastic.NewTermQuery("name", name) + err := db.GetElastic().Delete(query, "hardware_data") + if err != nil { return err } return nil -} \ No newline at end of file +} diff --git a/model/port.go b/model/port.go index 9dfbe7e..1573c10 100644 --- a/model/port.go +++ b/model/port.go @@ -2,7 +2,6 @@ package model import ( "background/db" - "reflect" "strings" "ubntgo/logger" @@ -13,30 +12,30 @@ import ( ) type Field struct { - Field string `sql:"Field"` - Type string `sql:"Type"` - Key string `sql:"Key"` + Field string `sql:"Field"` + Type string `sql:"Type"` + Key string `sql:"Key"` } - -func MysqlToElasticSearchMapping(types string,Key string) string{ - if Key == "PRI"{ +func MysqlToElasticSearchMapping(types string, Key string) string { + if Key == "PRI" { return "keyword" } - if(strings.Contains(types,"int(")){ + if strings.Contains(types, "int(") { return "integer" } - if(strings.Contains(types,"longblob")){ + if strings.Contains(types, "longblob") { return "text" } - if(strings.Contains(types,"varchar")){ + if strings.Contains(types, "varchar") { return "text" } - if(strings.Contains(types,"datetime")){ + if strings.Contains(types, "datetime") { return "date" } return "" } + /* "settings":{ "number_of_shards":1, @@ -65,103 +64,101 @@ func MysqlToElasticSearchMapping(types string,Key string) string{ } */ // 不同类型db之间进行缓存 -func QueryDocument(query elastic.Query,limit int,offset int) ([]Hardware,error){ +func QueryDocument(query elastic.Query, limit int, offset int) ([]Hardware, error) { var ret []Hardware - data,ids,e := db.GetElastic().Query("doc",query,reflect.TypeOf(Hardware{}),limit,offset) - log.Print(data) - if nil != e{ - return nil,e + ids, e := db.GetElastic().Query("doc", query, ret, limit, offset) + if nil != e { + return nil, e } i := 0 - for _,v := range data{ - ret = append(ret,v.(Hardware)) - ret[i].ID = ids[i] + for _, v := range ids { + ret[i].ID = v i++ } - return ret,nil + return ret, nil } -func InsertDocToElaticSearch(doc Doc) error{ +func InsertDocToElaticSearch(doc Doc) error { matchPhraseQuery := elastic.NewMatchQuery("title", doc.Title) - existedHardware,e := QueryHardwares(matchPhraseQuery,10,0) - log.Print(e,existedHardware) + existedHardware, e := QueryHardwares(matchPhraseQuery, 10, 0) + log.Print(e, existedHardware) - for _,v := range existedHardware{ - if v.Name == doc.Title{ + for _, v := range existedHardware { + if v.Name == doc.Title { log.Print(v.ID) - return errors.New(200,"existed title") + return errors.New(200, "existed title") } } - e = db.GetElastic().Create("doc","0","",doc) - if nil != e{ + e = db.GetElastic().Create("doc", "0", "", doc) + if nil != e { log.Print(e.Error()) return e } return nil } -func PortDocumentToElasticsearch(tblname string ) error{ - ret,e := GetAllDocs() - if nil != e{ +func PortDocumentToElasticsearch(tblname string) error { + ret, e := GetAllDocs() + if nil != e { log.Print(e.Error()) } - insert,err := json.Marshal(ret) - if nil != err{ + insert, err := json.Marshal(ret) + if nil != err { log.Print(err) } log.Print(len(ret)) log.Print(string(insert)) columns := []Field{} - e = db.GetMysqlClient().Query2("describe " + tblname,&columns) - if nil != e{ + e = db.GetMysqlClient().Query2("describe "+tblname, &columns) + if nil != e { logger.Debug(e.Error()) return e } - if existed,_ := db.GetElastic().IndexExisted(tblname);existed{ - for _,v := range ret{ + if existed, _ := db.GetElastic().IndexExisted(tblname); existed { + for _, v := range ret { e := InsertDocToElaticSearch(v) - if nil != e{ + if nil != e { log.Print(e.Error()) } log.Print("index not existed , create " + tblname) } - }else{ + } else { props := map[string]interface{}{} mapping := map[string]interface{}{ - "settings":map[string]interface{}{ - "analysis":map[string]interface{}{ - "analyzer":map[string]interface{}{ - "default":map[string]interface{}{ + "settings": map[string]interface{}{ + "analysis": map[string]interface{}{ + "analyzer": map[string]interface{}{ + "default": map[string]interface{}{ "type": "smartcn", }, }, }, }, "mappings": map[string]interface{}{ - "properties":props, + "properties": props, }, } - for _,v := range columns{ - props[v.Field] = map[string]string{"type":MysqlToElasticSearchMapping(v.Type,v.Key)}; + for _, v := range columns { + props[v.Field] = map[string]string{"type": MysqlToElasticSearchMapping(v.Type, v.Key)} } - dat,e := json.Marshal(mapping) - if nil != e{ + dat, e := json.Marshal(mapping) + if nil != e { log.Print(e.Error()) } - e = db.GetElastic().CreateIndex(tblname,string(dat)) - if nil != e{ + e = db.GetElastic().CreateIndex(tblname, string(dat)) + if nil != e { log.Print(e.Error()) } log.Print(string(dat)) - for _,v := range ret{ + for _, v := range ret { e := InsertDocToElaticSearch(v) - if nil != e{ + if nil != e { log.Print(e.Error()) } } } return nil -} \ No newline at end of file +} diff --git a/test/portData_test.go b/test/portData_test.go index cde8562..83e0aae 100644 --- a/test/portData_test.go +++ b/test/portData_test.go @@ -6,6 +6,7 @@ import ( "background/logs" "background/model" "background/utils" + "fmt" "log" "reflect" "testing" @@ -19,6 +20,7 @@ func InitConfig() { log.Println(e.Error()) } } + func InitMysql() { c := config.GetMysqlConfig() if c == nil { @@ -27,6 +29,7 @@ func InitMysql() { db.Init() } } + func InitRedisConfig() { e := config.InitRedis() if nil != e { @@ -34,6 +37,7 @@ func InitRedisConfig() { return } } + func InitElasticSearch() { e := db.GetElastic().CreateIndex("hardware", model.HardwareTypeMapping()) if nil != e { @@ -77,18 +81,30 @@ func TestQueryDoc(t *testing.T) { db.InitELK() query := elastic.NewTermQuery("content", "title") - searchResult, titles, e := db.GetElastic().Query("doc", query, reflect.TypeOf(model.Doc{}), 10, 0) + x := []model.Doc{} + titles, e := db.GetElastic().Query("doc", query, &x, 10, 0) if nil != e { log.Print(e.Error()) } - log.Print(searchResult) + log.Print(x) log.Print(titles) } func TestChangeStructFieldThroughStruct(t *testing.T) { - type XX struct { - A int16 `json:"bb"` - B string `json: "cc" xml:"test"` - } + var arr1 = []int{1, 2, 3} + log.Print(reflect.ValueOf(&arr1).Kind()) + a0 := reflect.ValueOf(&arr1).Elem() + log.Print(a0.Kind()) + e0 := make([]reflect.Value, 0) + e0 = append(e0, reflect.ValueOf(100)) + e0 = append(e0, reflect.ValueOf(200)) + e0 = append(e0, reflect.ValueOf(300)) + e0 = append(e0, reflect.ValueOf(400)) + val_arr1 := reflect.Append(a0, e0...) + a0.Set(val_arr1) + fmt.Println("a0 is ", a0) + fmt.Println("arr1 is ", arr1) + fmt.Println(cap(e0)) + fmt.Println(cap(arr1)) }