WIP: Connection Muxing

INTERNALS.md

@@ -13,8 +13,14 @@ Go pot as the names suggests is surprisingly 🥁... written in go! It is a HTTP
 
 ## Components
 Go pot is made up of a few different components that come together to make the staller. Some of the more idiomatic components are:
-* **Staller**: A http handler that will stall for a request for a given amount of time. It gets a generator instance it will keep on calling for new data until just before the timeout it has been given is reached. At which point it will correctly terminate the response.
+* **Staller**: A special handler that will stall for a request for a given amount of time. It gets a generator instance it will keep on calling for new data until just before the timeout it has been given is reached. At which point it will correctly terminate the response.
 * **Generator**: A generator will provide an infinite stream of fake structured data. That can be serialized into a number of different formats.
 * **TimeoutWatcher**: The timeout watcher will keep track of how long a bot is willing to wait for a response. It will do this by watching when a given IP address disconnects. If it gets a few similar disconnects in a row it will assume that that is the maximum time a bot is willing to wait for a response and then give a time just under that to the staller.
 * **Cluster**: The cluster is a way of sharing information about how long bots are willing to wait for a response to other nodes in the cluster. It uses memberlist (go)
-* **Recast**: Recast is a way of restarting / reallocating IP addresses to avoid being blacklisted by connecting clients. It uses telemetry to see if stalling connections and moves to a different IP block if not. 
+* **Recast**: Recast is a way of restarting / reallocating IP addresses to avoid being blacklisted by connecting clients. It uses telemetry to see if stalling connections and moves to a different IP block if not. 
+* **Detect / Multi protocol listener** Detect aims to watch for traffic on a TCP listener and make a guess at which protocol data being sent down the pipe belongs to. It does this by.
+When a new connection is opened:
+ * Wait for some data to be sent by the client
+ * If no data is sent in X seconds begin to "probe" by sending different protocol headers back down the pipe
+ * Wait for data while probes are sent 
+ * If still no data is sent change to the fallback handler

README.md

@@ -1,14 +1,14 @@
 # go-pot 🍯
-A HTTP tarpit written in Go designed to maximize bot misery through very slowly feeding them an infinite stream of fake secrets. 
+A Multi Protocol tarpit written in Go designed to maximize bot misery through very slowly feeding them an infinite stream of fake secrets. 
 
 <img src="docs/img/gopher.png" width="400px" />
 
 ## Features
 - **Realistic output**: Go pot will respond to requests with an infinite stream of realistic looking, parseable structured data full of fake secrets. `xml`, `json`, `yaml`, `hcl`, `toml`, `csv`, `ini`, and `sql` are all supported.
 - **Multiple protocols**: Both `http` and `ftp` are supported out of the box. Each with a tailored implementation. *More protocols are planned.*
 - **Intelligent stalling**: Go pot will attempt to work out how long a bot is willing to wait for a response and stall for exactly that long. This is done gradually making requests slower and slower until a timeout is reached. (Or the bot hangs forever!)
-- **Small Profile**: Go pot can run on extremely low resource machines and is designed to be as lightweight as possible.
-- **Clustering Support**: Go pot can be run in a clustered mode where multiple instances can share information about how long bots are willing to wait for a response. Also in cluster mode nodes can be configured to restart / reallocate IP addresses to avoid being blacklisted by connecting clients.
+- **Small Profile**: Go pot aims to target fairly low end hardware.
+- **Clustering Support**: Go pot can be run in a clustered mode where multiple instances can share information about how long bots are willing to wait for a response. Also in cluster mode nodes can be configured to restart / reallocate IP addresses to avoid being blacklisted by connecting clients. (Currently tested on AWS ECS)
 - **Customizable**: Go pot can be customized to respond with different different response times.
 
 ## Installation

cmd/ftp.go

@@ -23,7 +23,10 @@ var ftpCommand = &cobra.Command{
 
 		// Make sure only the FTP server is enabled
 		conf.FtpServer.Enabled = true
+
+		// Disable the server and multi protocol
 		conf.Server.Disable = true
+		conf.MultiProtocol.Enabled = false
 
 		di := di.CreateContainer(conf)
 		di.Run()

cmd/http.go

@@ -21,9 +21,12 @@ var httpCommand = &cobra.Command{
 			os.Exit(1)
 		}
 
+		// Enable the server
+		conf.Server.Disable = false
+
 		// Make sure only the HTTP server is enabled
 		conf.FtpServer.Enabled = false
-		conf.Server.Disable = false
+		conf.MultiProtocol.Enabled = false
 
 		di := di.CreateContainer(conf)
 		di.Run()

config/config.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"fmt"
 	"strings"
 
 	"github.com/knadh/koanf/providers/env"
@@ -14,6 +15,7 @@ type (
 	Config struct {
 		Server         serverConfig         `koanf:"server"`
 		FtpServer      ftpServerConfig      `koanf:"ftp_server"`
+		MultiProtocol  multiProtocolConfig  `koanf:"multi_protocol"`
 		Logging        loggingConfig        `koanf:"logging"`
 		Cluster        clusterConfig        `koanf:"cluster"`
 		TimeoutWatcher timeoutWatcherConfig `koanf:"timeout_watcher"`
@@ -22,13 +24,28 @@ type (
 		Staller        stallerConfig        `koanf:"staller"`
 	}
 
+	// Multi protocol configuration
+	multiProtocolConfig struct {
+		// If the multi protocol server should be started or not
+		Enabled bool `koanf:"enabled"`
+
+		// The port to listen on.
+		Port int `koanf:"port" validate:"required,min=1,max=65535,no_duplicate_port"`
+
+		// The protocol detectors to use
+		Host string `koanf:"host" validate:"omitempty"`
+
+		// The protocol detectors to use
+		Protocols []string `koanf:"protocols" validate:"required,dive,oneof=http ftp all"`
+	}
+
 	// Server specific configuration
 	serverConfig struct {
 		// If the http server should be disabled
 		Disable bool `koanf:"disable"`
 
 		// Server port to listen on
-		Port int `koanf:"port" validate:"required,min=1,max=65535"`
+		Port int `koanf:"port" validate:"required,min=1,max=65535,no_duplicate_port"`
 
 		// Server host to listen on
 		Host string `koanf:"host" validate:"required"`
@@ -74,13 +91,13 @@ type (
 
 		// The port to listen on N.b this is the control port
 		// port 20 is used for data transfer by default in active mode.
-		Port int `koanf:"port" validate:"required,min=1,max=65535"`
+		Port int `koanf:"port" validate:"required,min=1,max=65535,no_duplicate_port"`
 
 		// Host to listen on
 		Host string `koanf:"host" validate:"required"`
 
 		// Lower bound of ports exposed for passive mode default 50000-50100
-		PassivePortRange string `koanf:"passive_port_range" validate:"omitempty,port_range"`
+		PassivePortRange string `koanf:"passive_port_range" validate:"omitempty,port_range,no_duplicate_port_range"`
 
 		// The common name for the self signed certificate
 		CertCommonName string `koanf:"cert_common_name" validate:"omitempty"`
@@ -121,7 +138,7 @@ type (
 		Mode string `koanf:"mode" validate:"required_if=Enabled true,omitempty,oneof=fargate_ecs lan wan"`
 
 		// The bind address for the cluster to listen on
-		BindPort int `koanf:"bind_port" validate:"required_if=Enabled true,omitempty,min=1,max=65535"`
+		BindPort int `koanf:"bind_port" validate:"required_if=Enabled true,omitempty,min=1,max=65535,no_duplicate_port"`
 
 		// Known Peers
 		KnownPeerIps []string `koanf:"known_peer_ips" validate:"required_if=Mode lan Mode wan,omitempty"`
@@ -240,7 +257,7 @@ type (
 		Enabled bool `koanf:"enabled"`
 
 		// The port for the prometheus collection endpoint
-		Port int `koanf:"prometheus_port" validate:"required,min=1,max=65535"`
+		Port int `koanf:"prometheus_port" validate:"required,min=1,max=65535,no_duplicate_port"`
 
 		// The path for the prometheus endpoint
 		Path string `koanf:"prometheus_path" validate:"required"`
@@ -341,6 +358,29 @@ func NewConfig(cmd *cobra.Command, flagsUsed flagMap) (*Config, error) {
 	setStringSlice(k, "server.access_log.fields_to_log")
 	setStringSlice(k, "ftp_server.command_log.commands_to_log")
 	setStringSlice(k, "ftp_server.command_log.additional_fields")
+	setStringSlice(k, "multi_protocol.protocols")
+
+	// Implicitly enable each server type if specific configuration changes have been made to the default configuration
+	if err := setIfNotDefault(k, "ftp_server.enabled", true, map[string]interface{}{
+		"ftp_server.port": defaultConfig.FtpServer.Port,
+		"ftp_server.host": defaultConfig.FtpServer.Host,
+	}); err != nil {
+		return nil, err
+	}
+
+	if err := setIfNotDefault(k, "server.disable", false, map[string]interface{}{
+		"server.port": defaultConfig.Server.Port,
+		"server.host": defaultConfig.Server.Host,
+	}); err != nil {
+		return nil, err
+	}
+
+	if err := setIfNotDefault(k, "multi_protocol.enabled", false, map[string]interface{}{
+		"multi_protocol.port": defaultConfig.MultiProtocol.Port,
+		"multi_protocol.host": defaultConfig.MultiProtocol.Host,
+	}); err != nil {
+		return nil, err
+	}
 
 	var cfg *Config
 	if err := k.UnmarshalWithConf("", &cfg, koanf.UnmarshalConf{Tag: "koanf"}); err != nil {
@@ -378,3 +418,25 @@ func setStringSlice(k *koanf.Koanf, key string) {
 		k.Delete(key)
 	}
 }
+
+// In the event any of the given flags are not set to the given default value then the target key will be set to the target state
+func setIfNotDefault(k *koanf.Koanf, targetKey string, targetState interface{}, configToCheckForChanges map[string]interface{}) error {
+	// Assert that the desired key exists
+	if !k.Exists(targetKey) {
+		return fmt.Errorf("key %s does not exist when trying to set implicitly", targetKey)
+	}
+
+	// If the key is not the default value then we don't need to do anything
+	if k.Get(targetKey) == targetState {
+		return nil
+	}
+
+	for key, value := range configToCheckForChanges {
+		if k.Get(key) != value {
+			k.Set(targetKey, targetState)
+			return nil
+		}
+	}
+
+	return nil
+}

config/default.go

@@ -4,6 +4,12 @@ import "go.uber.org/zap/zapcore"
 
 // Default configuration values for the application
 var defaultConfig = Config{
+	MultiProtocol: multiProtocolConfig{
+		Enabled:   false,
+		Host:      "0.0.0.0",
+		Protocols: []string{"all"},
+		Port:      8081,
+	},
 	Server: serverConfig{
 		Disable:        false,
 		Port:           8080,

config/flags.go

@@ -222,6 +222,7 @@ var ftpFlags = flagMap{
 }
 
 var startFlags = flagMap{
+	// @todo - Next Major release [Swap this to be enabled]
 	"http-disabled": {
 		flagName:     "http-disabled",
 		configKey:    "server.disable",
@@ -237,6 +238,36 @@ var startFlags = flagMap{
 		configType:   "bool",
 		defaultValue: defaultConfig.FtpServer.Enabled,
 	},
+
+	// Multi-protocol options
+	"multi-protocol": {
+		flagName:     "multi-protocol",
+		configKey:    "multi_protocol.enabled",
+		description:  "Allows for multiple honeypots to bind to the same pot (will override --[protocol]-enabled / --[protocol]-disabled) flags",
+		configType:   "bool",
+		defaultValue: defaultConfig.MultiProtocol.Enabled,
+	},
+	"multi-protocol-port": {
+		flagName:     "multi-protocol-port",
+		configKey:    "multi_protocol.port",
+		description:  "The port to use for the multi protocol. Default(8081). Has no effect unless --multi-protocol specified",
+		configType:   "int",
+		defaultValue: defaultConfig.MultiProtocol.Port,
+	},
+	"multi-protocol-host": {
+		flagName:     "multi-protocol-host",
+		configKey:    "multi_protocol.host",
+		description:  "The host to bin the multi protocol listener to",
+		configType:   "string",
+		defaultValue: defaultConfig.MultiProtocol.Host,
+	},
+	"multi-protocol-protocols": {
+		flagName:     "multi-protocol-protocols",
+		configKey:    "multi_protocol.protocols",
+		description:  "Comma separated list of protocols to enable as part of the 'multi-protocol' listener. Can be one of 'ftp', 'http' or 'all'",
+		configType:   "string",
+		defaultValue: strings.Join(defaultConfig.MultiProtocol.Protocols, ","),
+	},
 }
 
 func GetStartFlags() flagMap {

config/validation.go

@@ -10,6 +10,76 @@ import (
 
 var portRangeRegex = regexp.MustCompile(`^(\d+)-(\d+)$`)
 
+type (
+	duplicatePortValidatorMeta struct {
+		name  string
+		ports []int
+	}
+
+	// Stateful validator use to assert all ports with the "no_duplicate_port" and "no_duplicate_port_range" tags are unique to stop later bind conflicts
+	duplicatePortValidator struct {
+		registeredPorts      map[int]*duplicatePortValidatorMeta
+		registeredPortRanges []*duplicatePortValidatorMeta
+	}
+)
+
+func NewDuplicatePortValidator() *duplicatePortValidator {
+	return &duplicatePortValidator{
+		registeredPorts:      make(map[int]*duplicatePortValidatorMeta),
+		registeredPortRanges: make([]*duplicatePortValidatorMeta, 0),
+	}
+}
+
+func (v *duplicatePortValidator) ValidatePort(fl validator.FieldLevel) bool {
+	port := int(fl.Field().Int())
+	if _, ok := v.registeredPorts[port]; ok {
+		return false
+	}
+
+	for _, portRange := range v.registeredPortRanges {
+		if port >= portRange.ports[0] && port <= portRange.ports[1] {
+			return false
+		}
+	}
+
+	// Register the port
+	v.registeredPorts[port] = &duplicatePortValidatorMeta{
+		name:  fl.FieldName(),
+		ports: []int{port},
+	}
+
+	return true
+}
+
+func (v *duplicatePortValidator) ValidatePortRange(fl validator.FieldLevel) bool {
+	minPort, maxPort, err := ParsePortRange(fl.Field().String())
+	if err != nil {
+		return false
+	}
+
+	// Assert there are no conflicts with existing port ranges
+	for _, port := range v.registeredPortRanges {
+		if max(port.ports[0], minPort) <= min(port.ports[1], maxPort) {
+			return false
+		}
+	}
+
+	// Assert there are no conflicts with existing ports
+	for port, _ := range v.registeredPorts {
+		if port >= minPort && port <= maxPort {
+			return false
+		}
+	}
+
+	// Register the port range
+	v.registeredPortRanges = append(v.registeredPortRanges, &duplicatePortValidatorMeta{
+		name:  fl.Param(),
+		ports: []int{minPort, maxPort},
+	})
+
+	return true
+}
+
 func ParsePortRange(portRange string) (int, int, error) {
 	matches := portRangeRegex.FindStringSubmatch(portRange)
 	if len(matches) < 3 {
@@ -51,5 +121,15 @@ func newConfigValidator() (*validator.Validate, error) {
 	if err := v.RegisterValidation("port_range", validatePortRange); err != nil {
 		return nil, err
 	}
+
+	dpv := NewDuplicatePortValidator()
+	if err := v.RegisterValidation("no_duplicate_port", dpv.ValidatePort); err != nil {
+		return nil, err
+	}
+
+	if err := v.RegisterValidation("no_duplicate_port_range", dpv.ValidatePortRange); err != nil {
+		return nil, err
+	}
+
 	return v, nil
 }