Add predefined field whitelist validation and parallel processing for dump

HISTORY.md

@@ -1,3 +1,11 @@
+## v0.2.29 add predefined field whitelist validation and parallel processing for dump
+
+-   create official field whitelists (ValidFieldsAll and ValidFieldsNext) in fields.go.
+-   add validation logic (ValidateFieldsAll, ValidateFieldsNext) and integrated it into the search, dump, and random commands.
+-   enhance the `dump` command with a worker pool for concurrency and thread-safe data writing mechanics.
+-   include comprehensive unit test coverage for the validation and concurrency logic in fields_test.go and dump_test.go.
+-   update README.md and README_ZH.md to natively reference the official FOFA API documentation URLs for valid fields.
+
 ## v0.2.28 fix dedup mode
 
 -   fix dedup of multiple fields to remove duplicates 

README.md

@@ -136,10 +136,10 @@ categories:
 
 | Parameter   | Abbreviation | Default Value | Description                                               |
 |-------------|--------------|---------------|-----------------------------------------------------------|
-| fields      | f            | ip,port       | Fields returned by FOFA. [Learn More](https://fofa.info/vip) |
+| fields      | f            | ip,port       | Fields returned by FOFA, valid fields refer to https://en.fofa.info/api |
 | format      |              | csv           | Output format: csv/json/xml                               |
 | outFile     | o            |               | Output file. If not set, prints to terminal               |
-| size        | s            | 100           | Query size. Maximum is 10,000, subject to `deductMode`    |
+| size        | s            | 100           | Query size. Maximum is 10,000, subject to `deductMode` *1*2 |
 | deductMode  |              |               | Consumption of f-points. If not set, uses max free query limit |
 | fixUrl      |              | false         | Combines URLs (e.g., 1.1.1.1,80 becomes http://1.1.1.1)   |
 | urlPrefix   |              | http://       | URL prefix                                                |
@@ -156,22 +156,30 @@ categories:
 | headline    |              | false         | Outputs CSV headers. Available only when format is CSV    |
 | help        | h            | false         | Displays usage information                                |
 
+*1: When the query contains `cert` and `banner`, the maximum results size setting is 2000 per page.
+*2: When the query contains `body`, the maximum results size setting is 500 per page.
+
 ### `dump`
 
 | Parameter   | Abbreviation | Default Value | Description                                               |
 |-------------|--------------|---------------|-----------------------------------------------------------|
-| fields      | f            | ip,port       | Fields returned by FOFA. [Learn More](https://fofa.info/vip) |
+| fields      | f            | ip,port       | Fields returned by FOFA, valid fields refer to https://en.fofa.info/api, for dump command refer to https://en.fofa.info/api/batches_pages |
 | format      |              | csv           | Output format: csv/json/xml                               |
 | outFile     | o            |               | Output file. If not set, prints to terminal               |
 | inFile      | i            |               | Input file. If not set, reads from pipeline input         |
-| size        | s            | 100           | Query size. No upper limit but consumes f-points or free query quota |
+| size        | s            | 100           | Query size. No upper limit but consumes f-points or free quota *1*2 |
 | fixUrl      |              | false         | Combines URLs (e.g., 1.1.1.1,80 becomes http://1.1.1.1)   |
 | urlPrefix   |              | http://       | URL prefix                                                |
 | full        |              | false         | Retrieves full data                                       |
 | batchSize   | bs           | 1000          | Number of records to fetch per batch                     |
 | batchType   | bt           |               | Batch query type: ip/domain                              |
+| workers     |              | 10            | Number of threads, defaults to 10 when using -i           |
+| rate        |              | 2             | Query rate per second                                     |
 | help        | h            | false         | Displays usage information                                |
 
+*1: When the query contains `cert` and `banner`, the maximum results size setting is 2000 per page.
+*2: When the query contains `body`, the maximum results size setting is 500 per page.
+
 ### `jsRender`
 
 | Parameter   | Abbreviation | Default Value | Description                                               |
@@ -254,13 +262,16 @@ categories:
 |-------------|--------------|--------------------------------------|------------------------------------------|
 | fields      | f            | ip,port,host,header,title,server,lastupdatetime | Fields returned by FOFA. [Learn More](https://en.fofa.info/vip) |
 | format      |              | json                                | Output format: csv/json/xml              |
-| size        | s            | 1                                   | Query count. `-1` for infinite queries   |
+| size        | s            | 1                                   | Query count. `-1` for infinite queries *1*2 |
 | sleep       |              | 1000                                | Interval between queries in milliseconds |
 | fixUrl      |              | false                               | Combines URLs (e.g., 1.1.1.1,80 becomes http://1.1.1.1) |
 | urlPrefix   |              | http://                             | URL prefix                              |
 | full        |              | false                               | Retrieves full data                     |
 | help        | h            | false                               | Displays usage information               |
 
+*1: When the query contains `cert` and `banner`, the maximum results size setting is 2000 per page.
+*2: When the query contains `body`, the maximum results size setting is 500 per page.
+
 ### `count`
 
 | Parameter   | Abbreviation | Default Value | Description                        |

README_ZH.md

@@ -139,10 +139,10 @@ categories:
 
 | 参数        | 参数简写 | 默认值  | 简介                                              |
 | ----------- | -------- | ------- | ------------------------------------------------- |
-| fields      | f        | ip,port | FOFA返回的字段选择，[了解更多](https://fofa.info/vip) |                             
+| fields      | f        | ip,port | FOFA返回的字段选择，有效字段参考https://fofa.info/api |                             
 | format      |          | csv     | 输出格式，可以为csv/json/xml                      |
 | outFile     | o        |         | 输出文件，如果不设置则终端打印                    |
-| size        | s        | 100     | 查询数量，最大为10000，受deductMode参数限制       |
+| size        | s        | 100     | 查询数量，最大为10000，受deductMode参数限制 *1*2  |
 | deductMode  |          |         | 消费f点数，不设置则读取用户最大免费数量           |
 | fixUrl      |          | false   | 是否组合url，例如1.1.1.1,80组合为http://1.1.1.1   |
 | urlPrefix   |          | http:// | url前缀                                           |
@@ -159,22 +159,30 @@ categories:
 | headline    |          | false   | 是否输出csv头，只有在format为csv时可用            |
 | help        | h        | false   | 使用方法                                          |
 
+*1：当获取字段包含 `cert` 和 `banner` 时，单次查询 size 最大支持 2000。
+*2：当获取字段包含 `body` 时，单次查询 size 最大支持 500。
+
 ### dump
 
 | 参数      | 参数简写 | 默认值  | 简介                                                  |
 | --------- | -------- | ------- | ----------------------------------------------------- |
-| fields    | f        | ip,port | FOFA返回的字段选择，[了解更多](https://fofa.info/vip) |
+| fields    | f        | ip,port | FOFA返回的字段选择，有效字段参考https://fofa.info/api，dump的参考https://fofa.info/api/batches_pages |
 | format    |          | csv     | 输出格式，可以为csv/json/xml                          |
 | outFile   | o        |         | 输出文件，如果不设置则终端打印                        |
 | inFile    | i        |         | 输入文件，如果不设置则读取管道输入                    |
-| size      | s        | 100     | 查询数量，无上限，但要扣除f点或免费数量               |
+| size      | s        | 100     | 查询数量，无上限，但要扣除f点或免费数量 *1*2          |
 | fixUrl    |          | false   | 是否组合url，例如1.1.1.1,80组合为http://1.1.1.1       |
 | urlPrefix |          | http:// | url前缀                                               |
 | full      |          | false   | 是否调取全量数据                                      |
 | batchSize | bs       | 1000    | 每次拉取多少条数据                                    |
 | batchType | bt       |         | 批量查询，可以为ip/domain                             |
+| workers   |          | 10      | 线程数量，当使用-i时默认10                            |
+| rate      |          | 2       | 每秒查询次数                                          |
 | help      | h        | false   | 使用方法                                              |
 
+*1：当获取字段包含 `cert` 和 `banner` 时，单次查询 size 最大支持 2000。
+*2：当获取字段包含 `body` 时，单次查询 size 最大支持 500。
+
 ### jsRender
 
 | 参数    | 参数简写 | 默认值 | 简介                               |
@@ -256,13 +264,16 @@ categories:
 | --------- | -------- | ----------------------------------------------- | ----------------------------------------------------- |
 | fields    | f        | ip,port,host,header,title,server,lastupdatetime | FOFA返回的字段选择，[了解更多](https://fofa.info/vip) |
 | format    |          | json                                            | 输出格式，可以为csv/json/xml                          |
-| size      | s        | 1                                               | 查询次数，-1表示永远不停                              |
+| size      | s        | 1                                               | 查询次数，-1表示永远不停 *1*2                         |
 | sleep     |          | 1000                                            | 获取间隔，单位ms                                      |
 | fixUrl    |          | false                                           | 是否组合url，例如1.1.1.1,80组合为http://1.1.1.1       |
 | urlPrefix |          | http://                                         | url前缀                                               |
 | full      |          | false                                           | 是否调取全量数据                                      |
 | help      | h        | false                                           | 使用方法                                              |
 
+*1：当获取字段包含 `cert` 和 `banner` 时，单次查询 size 最大支持 2000。
+*2：当获取字段包含 `body` 时，单次查询 size 最大支持 500。
+
 ### count
 
 | 参数 | 参数简写 | 默认值 | 简介     |

browser.go

@@ -4,25 +4,29 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"log"
+	"strings"
+	"time"
+
 	"github.com/go-rod/rod"
 	"github.com/go-rod/rod/lib/devices"
 	"github.com/go-rod/rod/lib/launcher"
+	flags "github.com/go-rod/rod/lib/launcher/flags"
 	"github.com/go-rod/rod/lib/proto"
 	"golang.org/x/net/html"
-	"log"
-	"strings"
-	"time"
 )
 
 type WorkerBrowser struct {
-	Url   string
-	Retry int
+	Url          string
+	Retry        int
+	LauncherArgs []string
 }
 
 func NewWorkerBrowser(url string, retry int) *WorkerBrowser {
 	return &WorkerBrowser{
-		Url:   url,
-		Retry: retry,
+		Url:          url,
+		Retry:        retry,
+		LauncherArgs: []string{},
 	}
 }
 
@@ -143,6 +147,10 @@ func (wp *WorkerBrowser) renderScan(url string) (string, error) {
 			l.Set("--ignore-certificate-errors")
 			l.Set("disable-notifications", "true")
 
+			for _, arg := range wp.LauncherArgs {
+				l.Set(flags.Flag(arg), "true")
+			}
+
 			lurl := l.MustLaunch()
 
 			b := rod.New().ControlURL(lurl).

browser_test.go

@@ -2,11 +2,12 @@ package gofofa
 
 import (
 	"fmt"
-	"github.com/stretchr/testify/assert"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 	"time"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func runningTime() func() {
@@ -80,18 +81,21 @@ func TestWorkerBrowser_Run(t *testing.T) {
 
 	// 错误url情况
 	b := NewWorkerBrowser("", 3)
+	b.LauncherArgs = []string{"no-sandbox"}
 	body, err := b.Run()
 	assert.NotNil(t, err)
 	assert.Nil(t, body["body"])
 
 	// 常规js渲染
 	b = NewWorkerBrowser(ts.URL+"/js/normal", 3)
+	b.LauncherArgs = []string{"no-sandbox"}
 	body, err = b.Run()
 	assert.Nil(t, err)
 	assert.Equal(t, "Updated Title", body["title"])
 
 	// 页面跳转
 	b = NewWorkerBrowser(ts.URL+"/js/redirect", 3)
+	b.LauncherArgs = []string{"no-sandbox"}
 	body, err = b.Run()
 	assert.Nil(t, err)
 	assert.Equal(t, "Successfully Title", body["title"])

cmd/fofa/cmd/dump.go

@@ -2,16 +2,20 @@ package cmd
 
 import (
 	"bufio"
+	"context"
 	"errors"
 	"fmt"
-	"github.com/FofaInfo/GoFOFA"
-	"github.com/FofaInfo/GoFOFA/pkg/outformats"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli/v2"
 	"io"
 	"log"
 	"os"
 	"strings"
+	"sync"
+
+	gofofa "github.com/FofaInfo/GoFOFA"
+	"github.com/FofaInfo/GoFOFA/pkg/outformats"
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/time/rate"
 )
 
 var (
@@ -112,6 +116,18 @@ var dumpCmd = &cli.Command{
 			Usage:       "use custom fields",
 			Destination: &customFields,
 		},
+		&cli.IntFlag{
+			Name:        "workers",
+			Value:       1,
+			Usage:       "number of workers",
+			Destination: &workers,
+		},
+		&cli.IntFlag{
+			Name:        "rate",
+			Value:       2,
+			Usage:       "fofa query per second",
+			Destination: &ratePerSecond,
+		},
 	},
 	Action: DumpAction,
 }
@@ -171,6 +187,14 @@ func DumpAction(ctx *cli.Context) error {
 		if err = scanner.Err(); err != nil {
 			log.Fatal(err)
 		}
+
+		// 默认workers提升到10，rate提升到2
+		if !ctx.IsSet("workers") {
+			workers = 10
+		}
+		if !ctx.IsSet("rate") {
+			ratePerSecond = 2
+		}
 	}
 
 	if len(queries) == 0 {
@@ -190,6 +214,11 @@ func DumpAction(ctx *cli.Context) error {
 		return errors.New("fofa fields cannot be empty")
 	}
 
+	// 字段白名单前置强校验 (使用 ValidateFieldsNext)
+	if err := gofofa.ValidateFieldsNext(fields); err != nil {
+		return err
+	}
+
 	// headline只允许在format=csv的情况下使用
 	if headline && format != "csv" && len(outFile) > 0 {
 		return errors.New("headline param is only allowed if format is csv, outFile not be empty")
@@ -249,26 +278,52 @@ func DumpAction(ctx *cli.Context) error {
 	}
 
 	// do search
-	for _, query := range queries {
-		log.Println("dump data of query:", query)
+	var locker sync.Mutex
+	wg := sync.WaitGroup{}
+	queriesChan := make(chan string, len(queries))
+	limiter := rate.NewLimiter(rate.Limit(ratePerSecond), 5)
 
-		fetchedSize := 0
-		err := fofaCli.DumpSearch(query, size, batchSize, fields, func(res [][]string, allSize int) (err error) {
-			fetchedSize += len(res)
-			log.Printf("size: %d/%d, %.2f%%", fetchedSize, allSize, 100*float32(fetchedSize)/float32(allSize))
-			// output
-			err = writer.WriteAll(res)
-			return err
-		}, gofofa.SearchOptions{
-			FixUrl:    fixUrl,
-			UrlPrefix: urlPrefix,
-			Full:      full,
-		})
-		if err != nil {
-			log.Println("fetch error:", err)
-			//return err
+	worker := func(qChan <-chan string, wg *sync.WaitGroup) {
+		for query := range qChan {
+			if err := limiter.Wait(context.Background()); err != nil {
+				fmt.Println("Error: ", err)
+			}
+			log.Println("dump data of query:", query)
+
+			fetchedSize := 0
+			err := fofaCli.DumpSearch(query, size, batchSize, fields, func(res [][]string, allSize int) (err error) {
+				fetchedSize += len(res)
+				log.Printf("size: %d/%d, %.2f%% for query: %s", fetchedSize, allSize, 100*float32(fetchedSize)/float32(allSize), query)
+				// output
+				locker.Lock()
+				defer locker.Unlock()
+				err = writer.WriteAll(res)
+				if err == nil {
+					writer.Flush()
+				}
+				return err
+			}, gofofa.SearchOptions{
+				FixUrl:    fixUrl,
+				UrlPrefix: urlPrefix,
+				Full:      full,
+			})
+			if err != nil {
+				log.Printf("fetch error for query %s: %v\n", query, err)
+			}
+			wg.Done()
 		}
 	}
 
+	for w := 0; w < workers; w++ {
+		go worker(queriesChan, &wg)
+	}
+
+	for _, query := range queries {
+		wg.Add(1)
+		queriesChan <- query
+	}
+	close(queriesChan)
+	wg.Wait()
+
 	return nil
 }