feat: External Migration

fix: File path error

config_example/rules/CAPTCHA.yml

@@ -1,4 +0,0 @@
-secret_key: "0378b0f84c4310279918d71a5647ba5d"
-captcha_validate_time: 600
-captcha_challenge_session_timeout: 120
-hcaptcha_secret: ""

config_example/rules/HTTPFlood.yml

@@ -1,4 +0,0 @@
-HTTPFloodSpeedLimit:
-  - "150/10s"
-HTTPFloodSameURILimit:
-  - "50/10s"

config_example/rules/Server.yml

@@ -0,0 +1,22 @@
+CAPTCHA:
+  secret_key: "0378b0f84c4310279918d71a5647ba5d"
+  captcha_validate_time: 600
+  captcha_challenge_session_timeout: 120
+  hcaptcha_secret: ""
+HTTPFlood:
+  HTTPFloodSpeedLimit:
+    - "150/10s"
+  HTTPFloodSameURILimit:
+    - "50/10s"
+VerifyBot:
+  verify_google_bot: true
+  verify_bing_bot: true
+  verify_baidu_bot: true
+  verify_yandex_bot: true
+  verify_sogou_bot: true
+  verify_apple_bot: true
+ExternalMigration:
+  enabled: false
+  redirect_url: "https://example.com/migration"
+  secret_key: "0378b0f84c4310279918d71a5647ba5d"
+  session_timeout: 1800

config_example/rules/VerifyBot.yml

@@ -1,6 +0,0 @@
-verify_google_bot: true
-verify_bing_bot: true
-verify_baidu_bot: true
-verify_yandex_bot: true
-verify_sogou_bot: true
-verify_apple_bot: true

internal/check/ExternalMigration.go

@@ -0,0 +1,103 @@
+package check
+
+import (
+	"crypto/hmac"
+	"crypto/sha512"
+	"fmt"
+	"server_torii/internal/action"
+	"server_torii/internal/config"
+	"server_torii/internal/dataType"
+	"server_torii/internal/utils"
+	"strconv"
+	"strings"
+	"time"
+)
+
+func ExternalMigration(reqData dataType.UserRequest, ruleSet *config.RuleSet, decision *action.Decision, sharedMem *dataType.SharedMemory) {
+	if !ruleSet.ExternalMigrationRule.Enabled {
+		decision.Set(action.Continue)
+		return
+	}
+
+	if !verifyExternalMigrationClearanceCookie(reqData, *ruleSet) {
+		decision.SetResponse(action.Done, []byte("EXTERNAL"), genExternalMigrationSessionID(reqData, *ruleSet))
+		return
+	}
+
+	decision.Set(action.Continue)
+}
+
+func GenExternalMigrationClearance(reqData dataType.UserRequest, ruleSet config.RuleSet) []byte {
+	timeNow := time.Now().Unix()
+	mac := hmac.New(sha512.New, []byte(ruleSet.ExternalMigrationRule.SecretKey))
+	mac.Write([]byte(fmt.Sprintf("%d%s%sEXTERNAL-CLEARANCE", timeNow, reqData.Host, utils.GetClearanceUserAgent(reqData.UserAgent))))
+	return []byte(fmt.Sprintf("%s:%s", fmt.Sprintf("%d", time.Now().Unix()), fmt.Sprintf("%x", mac.Sum(nil))))
+}
+
+func verifyExternalMigrationClearanceCookie(reqData dataType.UserRequest, ruleSet config.RuleSet) bool {
+	if reqData.ToriiClearance == "" {
+		return false
+	}
+	parts := strings.Split(reqData.ToriiClearance, ":")
+	if len(parts) != 2 {
+		return false
+	}
+	timestamp := parts[0]
+	expectedHash := parts[1]
+
+	timeNow := time.Now().Unix()
+	parsedTimestamp, err := strconv.ParseInt(timestamp, 10, 64)
+	if err != nil {
+		utils.LogError(reqData, "", fmt.Sprintf("Error parsing timestamp: %v", err))
+		return false
+	}
+
+	if timeNow-parsedTimestamp > ruleSet.ExternalMigrationRule.SessionTimeout {
+		return false
+	}
+
+	mac := hmac.New(sha512.New, []byte(ruleSet.ExternalMigrationRule.SecretKey))
+	mac.Write([]byte(fmt.Sprintf("%d%s%sEXTERNAL-CLEARANCE", parsedTimestamp, reqData.Host, utils.GetClearanceUserAgent(reqData.UserAgent))))
+	computedHash := fmt.Sprintf("%x", mac.Sum(nil))
+
+	return hmac.Equal([]byte(computedHash), []byte(expectedHash))
+
+}
+
+func genExternalMigrationSessionID(reqData dataType.UserRequest, ruleSet config.RuleSet) []byte {
+	timeNow := time.Now().Unix()
+	mac := hmac.New(sha512.New, []byte(ruleSet.ExternalMigrationRule.SecretKey))
+	mac.Write([]byte(fmt.Sprintf("%d%s%sEXTERNAL-SESSION", timeNow, reqData.Host, utils.GetClearanceUserAgent(reqData.UserAgent))))
+	return []byte(fmt.Sprintf("%s:%s", fmt.Sprintf("%d", time.Now().Unix()), fmt.Sprintf("%x", mac.Sum(nil))))
+}
+
+func VerifyExternalMigrationSessionIDCookie(reqData dataType.UserRequest, ruleSet config.RuleSet) bool {
+	if reqData.ToriiSessionID == "" {
+		return false
+	}
+	parts := strings.Split(reqData.ToriiSessionID, ":")
+	if len(parts) != 2 {
+		return false
+	}
+	timestamp := parts[0]
+	expectedHash := parts[1]
+
+	parsedTimestamp, err := strconv.ParseInt(timestamp, 10, 64)
+	if err != nil {
+		utils.LogError(reqData, "", fmt.Sprintf("Error parsing timestamp: %v", err))
+		return false
+	}
+
+	mac := hmac.New(sha512.New, []byte(ruleSet.ExternalMigrationRule.SecretKey))
+	mac.Write([]byte(fmt.Sprintf("%d%s%sEXTERNAL-SESSION", parsedTimestamp, reqData.Host, utils.GetClearanceUserAgent(reqData.UserAgent))))
+	computedHash := fmt.Sprintf("%x", mac.Sum(nil))
+
+	return hmac.Equal([]byte(computedHash), []byte(expectedHash))
+
+}
+
+func CalculateExternalMigrationHMAC(sessionID, timestampStr, secretKey string) string {
+	mac := hmac.New(sha512.New, []byte(secretKey))
+	mac.Write([]byte(fmt.Sprintf("%s%s", sessionID, timestampStr)))
+	return fmt.Sprintf("%x", mac.Sum(nil))
+}

internal/config/config.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"bufio"
+	"fmt"
 	"gopkg.in/yaml.v3"
 	"net"
 	"os"
@@ -27,6 +28,20 @@ type MainConfig struct {
 
 // LoadMainConfig Read the configuration file and return the configuration object
 func LoadMainConfig(basePath string) (*MainConfig, error) {
+
+	defaultCfg := MainConfig{
+		Port:                           "25555",
+		WebPath:                        "/torii",
+		RulePath:                       "/www/server_torii/config/rules",
+		ErrorPage:                      "/www/server_torii/config/error_page",
+		LogPath:                        "/www/server_torii/log/",
+		NodeName:                       "Server Torii",
+		ConnectingHostHeaders:          []string{"Torii-Real-Host"},
+		ConnectingIPHeaders:            []string{"Torii-Real-IP"},
+		ConnectingURIHeaders:           []string{"Torii-Original-URI"},
+		ConnectingCaptchaStatusHeaders: []string{"Torii-Captcha-Status"},
+	}
+
 	exePath, err := os.Executable()
 	if err != nil {
 		return nil, err
@@ -38,12 +53,12 @@ func LoadMainConfig(basePath string) (*MainConfig, error) {
 
 	data, err := os.ReadFile(configPath)
 	if err != nil {
-		return nil, err
+		return &defaultCfg, err
 	}
 
 	var cfg MainConfig
 	if err := yaml.Unmarshal(data, &cfg); err != nil {
-		return nil, err
+		return &defaultCfg, err
 	}
 
 	return &cfg, nil
@@ -51,95 +66,115 @@ func LoadMainConfig(basePath string) (*MainConfig, error) {
 
 // RuleSet stores all rules
 type RuleSet struct {
-	IPAllowTrie   *dataType.TrieNode
-	IPBlockTrie   *dataType.TrieNode
-	URLAllowList  *dataType.URLRuleList
-	URLBlockList  *dataType.URLRuleList
-	CAPTCHARule   *dataType.CaptchaRule
-	VerifyBotRule *dataType.VerifyBotRule
-	HTTPFloodRule *dataType.HTTPFloodRule
+	IPAllowTrie           *dataType.TrieNode
+	IPBlockTrie           *dataType.TrieNode
+	URLAllowList          *dataType.URLRuleList
+	URLBlockList          *dataType.URLRuleList
+	CAPTCHARule           *dataType.CaptchaRule
+	VerifyBotRule         *dataType.VerifyBotRule
+	HTTPFloodRule         *dataType.HTTPFloodRule
+	ExternalMigrationRule *dataType.ExternalMigrationRule
+}
+
+// ruleSetWrapper
+type ruleSetWrapper struct {
+	CAPTCHARule           *dataType.CaptchaRule           `yaml:"CAPTCHA"`
+	VerifyBotRule         *dataType.VerifyBotRule         `yaml:"VerifyBot"`
+	HTTPFloodRule         httpFloodRuleWrapper            `yaml:"HTTPFlood"`
+	ExternalMigrationRule *dataType.ExternalMigrationRule `yaml:"ExternalMigration"`
+}
+
+type httpFloodRuleWrapper struct {
+	HTTPFloodSpeedLimit   []string `yaml:"HTTPFloodSpeedLimit"`
+	HTTPFloodSameURILimit []string `yaml:"HTTPFloodSameURILimit"`
 }
 
 // LoadRules Load all rules from the specified path
 func LoadRules(rulePath string) (*RuleSet, error) {
 	rs := RuleSet{
-		IPAllowTrie:   &dataType.TrieNode{},
-		IPBlockTrie:   &dataType.TrieNode{},
-		URLAllowList:  &dataType.URLRuleList{},
-		URLBlockList:  &dataType.URLRuleList{},
-		CAPTCHARule:   &dataType.CaptchaRule{},
-		VerifyBotRule: &dataType.VerifyBotRule{},
-		HTTPFloodRule: &dataType.HTTPFloodRule{},
+		IPAllowTrie:           &dataType.TrieNode{},
+		IPBlockTrie:           &dataType.TrieNode{},
+		URLAllowList:          &dataType.URLRuleList{},
+		URLBlockList:          &dataType.URLRuleList{},
+		CAPTCHARule:           &dataType.CaptchaRule{},
+		VerifyBotRule:         &dataType.VerifyBotRule{},
+		HTTPFloodRule:         &dataType.HTTPFloodRule{},
+		ExternalMigrationRule: &dataType.ExternalMigrationRule{},
 	}
 
 	// Load IP Allow List
-	ipAllowFile := rulePath + "/IP_AllowList.conf"
+	ipAllowFile := filepath.Join(rulePath, "/IP_AllowList.conf")
 	if err := loadIPRules(ipAllowFile, rs.IPAllowTrie); err != nil {
 		return nil, err
 	}
 
 	// Load IP Block List
-	ipBlockFile := rulePath + "/IP_BlockList.conf"
+	ipBlockFile := filepath.Join(rulePath, "/IP_BlockList.conf")
 	if err := loadIPRules(ipBlockFile, rs.IPBlockTrie); err != nil {
 		return nil, err
 	}
 
 	// Load URL Allow List
-	urlAllowFile := rulePath + "/URL_AllowList.conf"
+	urlAllowFile := filepath.Join(rulePath, "/URL_AllowList.conf")
 	if err := loadURLRules(urlAllowFile, rs.URLAllowList); err != nil {
 		return nil, err
 	}
 
 	// Load URL Block List
-	urlBlockFile := rulePath + "/URL_BlockList.conf"
+	urlBlockFile := filepath.Join(rulePath, "/URL_BlockList.conf")
 	if err := loadURLRules(urlBlockFile, rs.URLBlockList); err != nil {
 		return nil, err
 	}
 
-	// Load CAPTCHA Rule
-	captchaFile := rulePath + "/CAPTCHA.yml"
-	if err := loadCAPTCHARule(captchaFile, rs.CAPTCHARule); err != nil {
-		return nil, err
-	}
-
-	// Load Verify Bot Rule
-	verifyBotFile := rulePath + "/VerifyBot.yml"
-	if err := loadVerifyBotRule(verifyBotFile, rs.VerifyBotRule); err != nil {
-		return nil, err
-	}
-
-	// Load HTTP Flood Rule
-	httpFloodFile := rulePath + "/HTTPFlood.yml"
-	if err := loadHTTPFloodRule(httpFloodFile, rs.HTTPFloodRule); err != nil {
-		return nil, err
+	YAMLFile := filepath.Join(rulePath, "Server.yml")
+	set, err := loadServerRules(YAMLFile, rs)
+	if err != nil {
+		return set, err
 	}
 
 	return &rs, nil
 }
 
-func loadCAPTCHARule(file string, rule *dataType.CaptchaRule) error {
-	data, err := os.ReadFile(file)
+func loadServerRules(YAMLFile string, rs RuleSet) (*RuleSet, error) {
+	yamlData, err := os.ReadFile(YAMLFile)
 	if err != nil {
-		return err
+		if os.IsNotExist(err) {
+			return nil, fmt.Errorf("[ERROR] rules file %s does not exist: %w", YAMLFile, err)
+		} else {
+			return nil, fmt.Errorf("[ERROR] failed to read rules file %s: %w", YAMLFile, err)
+		}
 	}
 
-	if err := yaml.Unmarshal(data, &rule); err != nil {
-		return err
+	var wrapper ruleSetWrapper
+	if err := yaml.Unmarshal(yamlData, &wrapper); err != nil {
+		return nil, fmt.Errorf("[ERROR] failed to parse rules file %s: %w", YAMLFile, err)
 	}
 
-	return nil
-
-}
-
-func loadVerifyBotRule(file string, rule *dataType.VerifyBotRule) error {
-	data, err := os.ReadFile(file)
-	if err != nil {
-		return err
+	*rs.CAPTCHARule = *wrapper.CAPTCHARule
+	*rs.VerifyBotRule = *wrapper.VerifyBotRule
+	if wrapper.ExternalMigrationRule != nil {
+		*rs.ExternalMigrationRule = *wrapper.ExternalMigrationRule
 	}
-	if err := yaml.Unmarshal(data, &rule); err != nil {
-		return err
+
+	rs.HTTPFloodRule.HTTPFloodSpeedLimit = make(map[int64]int64)
+	rs.HTTPFloodRule.HTTPFloodSameURILimit = make(map[int64]int64)
+
+	for _, s := range wrapper.HTTPFloodRule.HTTPFloodSpeedLimit {
+		limit, seconds, err := utils.ParseRate(s)
+		if err != nil {
+			return nil, err
+		}
+		rs.HTTPFloodRule.HTTPFloodSpeedLimit[seconds] = limit
 	}
-	return nil
+
+	for _, s := range wrapper.HTTPFloodRule.HTTPFloodSameURILimit {
+		limit, seconds, err := utils.ParseRate(s)
+		if err != nil {
+			return nil, err
+		}
+		rs.HTTPFloodRule.HTTPFloodSameURILimit[seconds] = limit
+	}
+	return nil, nil
 }
 
 // loadIPRules read the IP rule file and insert the rules into the trie
@@ -217,43 +252,3 @@ func loadURLRules(filePath string, list *dataType.URLRuleList) error {
 
 	return scanner.Err()
 }
-
-func loadHTTPFloodRule(file string, rule *dataType.HTTPFloodRule) error {
-	data, err := os.ReadFile(file)
-	if err != nil {
-		return err
-	}
-
-	type httpFloodRuleYAML struct {
-		HTTPFloodSpeedLimit   []string `yaml:"HTTPFloodSpeedLimit"`
-		HTTPFloodSameURILimit []string `yaml:"HTTPFloodSameURILimit"`
-	}
-
-	var ymlRule httpFloodRuleYAML
-	err = yaml.Unmarshal(data, &ymlRule)
-	if err != nil {
-		return err
-	}
-
-	rule.HTTPFloodSpeedLimit = make(map[int64]int64)
-	rule.HTTPFloodSameURILimit = make(map[int64]int64)
-
-	for _, s := range ymlRule.HTTPFloodSpeedLimit {
-		limit, seconds, err := utils.ParseRate(s)
-		if err != nil {
-			return err
-		}
-		rule.HTTPFloodSpeedLimit[seconds] = limit
-	}
-
-	for _, s := range ymlRule.HTTPFloodSameURILimit {
-		limit, seconds, err := utils.ParseRate(s)
-		if err != nil {
-			return err
-		}
-		rule.HTTPFloodSameURILimit[seconds] = limit
-	}
-
-	return nil
-
-}

internal/dataType/type.go

@@ -31,6 +31,13 @@ type HTTPFloodRule struct {
 	HTTPFloodSameURILimit map[int64]int64
 }
 
+type ExternalMigrationRule struct {
+	Enabled        bool   `yaml:"enabled"`
+	RedirectUrl    string `yaml:"redirect_url"`
+	SecretKey      string `yaml:"secret_key"`
+	SessionTimeout int64  `yaml:"session_timeout"`
+}
+
 type SharedMemory struct {
 	HTTPFloodSpeedLimitCounter   *Counter
 	HTTPFloodSameURILimitCounter *Counter

internal/server/checker.go

@@ -25,6 +25,7 @@ func CheckMain(w http.ResponseWriter, userRequestData dataType.UserRequest, rule
 	checkFuncs = append(checkFuncs, check.URLBlockList)
 	checkFuncs = append(checkFuncs, check.VerifyBot)
 	checkFuncs = append(checkFuncs, check.HTTPFlood)
+	checkFuncs = append(checkFuncs, check.ExternalMigration)
 	checkFuncs = append(checkFuncs, check.Captcha)
 
 	for _, checkFunc := range checkFuncs {
@@ -106,6 +107,11 @@ func CheckMain(w http.ResponseWriter, userRequestData dataType.UserRequest, rule
 			return
 		}
 
+	} else if bytes.Compare(decision.HTTPCode, []byte("EXTERNAL")) == 0 {
+		w.Header().Set("Set-Cookie", "__torii_sessionid="+string(decision.ResponseData)+"; Path=/;  Max-Age=86400; Priority=High; HttpOnly; SameSite=Lax")
+		w.Header().Set("Location", ruleSet.ExternalMigrationRule.RedirectUrl+"?domain="+userRequestData.Host+"&session_id="+string(decision.ResponseData)+"&original_uri="+userRequestData.Uri)
+		w.WriteHeader(http.StatusFound)
+		return
 	} else {
 		//should never happen
 		utils.LogError(userRequestData, fmt.Sprintf("Error access in wrong state: %v", decision), "CheckMain")

internal/server/torii.go

@@ -2,6 +2,7 @@ package server
 
 import (
 	"bytes"
+	"crypto/hmac"
 	"html/template"
 	"net/http"
 	"server_torii/internal/action"
@@ -9,6 +10,7 @@ import (
 	"server_torii/internal/config"
 	"server_torii/internal/dataType"
 	"server_torii/internal/utils"
+	"strconv"
 	"time"
 )
 
@@ -20,6 +22,8 @@ func CheckTorii(w http.ResponseWriter, r *http.Request, reqData dataType.UserReq
 		check.CheckCaptcha(r, reqData, ruleSet, decision)
 	} else if reqData.Uri == cfg.WebPath+"/health_check" {
 		decision.SetResponse(action.Done, []byte("200"), []byte("ok"))
+	} else if reqData.Uri == cfg.WebPath+"/external_migration" {
+		handleExternalMigration(w, r, reqData, ruleSet, decision)
 	}
 	if bytes.Compare(decision.HTTPCode, []byte("200")) == 0 {
 		if bytes.Compare(decision.ResponseData, []byte("ok")) == 0 {
@@ -88,3 +92,44 @@ func CheckTorii(w http.ResponseWriter, r *http.Request, reqData dataType.UserReq
 		}
 	}
 }
+
+func handleExternalMigration(w http.ResponseWriter, r *http.Request, data dataType.UserRequest, set *config.RuleSet, decision *action.Decision) {
+	if !set.ExternalMigrationRule.Enabled {
+		decision.SetResponse(action.Done, []byte("200"), []byte("bad"))
+		return
+	}
+
+	originalURI := r.URL.Query().Get("original_uri")
+	timestampStr := r.URL.Query().Get("timestamp")
+	hmacParam := r.URL.Query().Get("hmac")
+
+	if timestampStr == "" || hmacParam == "" {
+		decision.SetResponse(action.Done, []byte("200"), []byte("bad"))
+		return
+	}
+
+	timestamp, err := strconv.ParseInt(timestampStr, 10, 64)
+	if err != nil {
+		decision.SetResponse(action.Done, []byte("200"), []byte("bad"))
+		return
+	}
+
+	currentTime := time.Now().Unix()
+	if currentTime-timestamp > set.ExternalMigrationRule.SessionTimeout {
+		decision.SetResponse(action.Done, []byte("200"), []byte("bad"))
+		return
+	}
+
+	if !check.VerifyExternalMigrationSessionIDCookie(data, *set) {
+		decision.SetResponse(action.Done, []byte("200"), []byte("badSession"))
+		return
+	}
+
+	expectedHMAC := check.CalculateExternalMigrationHMAC(data.ToriiSessionID, timestampStr, set.ExternalMigrationRule.SecretKey)
+	if !hmac.Equal([]byte(expectedHMAC), []byte(hmacParam)) {
+		decision.SetResponse(action.Done, []byte("200"), []byte("bad"))
+		return
+	}
+
+	decision.SetResponse(action.Continue, []byte("302"), []byte(originalURI))
+}

main.go

@@ -27,7 +27,7 @@ func main() {
 	// Load MainConfig
 	cfg, err := config.LoadMainConfig(basePath)
 	if err != nil {
-		log.Fatalf("Load config failed: %v", err)
+		log.Printf("[ERROR] Load config failed: %v. Using default config.", err)
 	}
 
 	// Load rules