edge-http: Server doesn't allow more clients

[Luni-4]

This issue might be related to #62, though I'm not certain. I'm on an esp32c3 and when I try to access an edge-http server from two different clients using curl, I encounter the following error:

curl: (7) Failed to connect to 192.168.178.142 port 3000 after 7 ms: Couldn't connect to server
Hello world!⏎

I get the correct answer only the first time, but only when timeouts are disabled. At other times, the result is the same above.

Hello world!⏎
Hello world!⏎

This is the script I have used to create the clients:

#!/bin/bash
for i in {1..2}; do
  curl -X GET http://192.168.178.142:3000 \
       -H "Connection: close" \
       -H "User-Agent: client$i" &
done
wait

Below the code

Toml

[package]
name = "example"
version = "0.1.0"
rust-version = "1.89"
edition = "2024"

[dependencies]
# Embassy framework
embassy-executor.version = "0.9.1"
embassy-executor.features = ["log"]

embassy-net.version = "0.7.1"
embassy-net.features = ["dhcpv4", "log", "medium-ethernet", "tcp", "udp"]

embassy-sync.version = "0.7.2"

embassy-time.version = "0.5.0"
embassy-time.features = ["log"]

embassy-embedded-hal = "0.5.0"

# HTTP server
edge-http.version = "0.6.1"
edge-http.git = "https://github.com/Luni-4/edge-net.git"
edge-http.branch = "deps"

# Networking traits
edge-nal.version = "0.5.0"
edge-nal.git = "https://github.com/Luni-4/edge-net.git"
edge-nal.branch = "deps"

edge-nal-embassy.version = "0.7.0"
edge-nal-embassy.git = "https://github.com/Luni-4/edge-net.git"
edge-nal-embassy.branch = "deps"

# Esp dependencies
esp-alloc.version = "0.9.0"

esp-bootloader-esp-idf.version = "0.4.0"
esp-bootloader-esp-idf.features = ["esp32c3", "log-04"]

esp-hal.version = "1.0.0"
esp-hal.features = ["esp32c3", "log-04", "unstable"]

esp-rtos.version = "0.2.0"
esp-rtos.features = ["embassy", "esp-alloc", "esp-radio", "esp32c3", "log-04"]

esp-println.version = "0.16.1"
esp-println.features = ["esp32c3", "log-04"]

esp-radio.version = "0.17.0"
esp-radio.features = ["esp-alloc", "esp32c3", "log-04", "wifi"]

# Logging utilities
log.version = "0.4.28"

# Asynchronous read and write traits
embedded-io-async.version = "0.7.0"

# Static cell
static_cell.version = "2.1.1"
static_cell.features = ["nightly"]

# Toml dependency
toml-cfg.version = "0.2.0"
toml-cfg.default-features = false

[build-dependencies]
toml-cfg.version = "0.2.0"
toml-cfg.default-features = false

[profile.dev]
opt-level = "s"

[profile.release]
codegen-units    = 1  
debug            = 2
debug-assertions = false
incremental      = false
lto              = 'fat'
opt-level        = 's'
overflow-checks  = false

Rust

#![no_std]
#![no_main]
#![deny(
    clippy::mem_forget,
    reason = "mem::forget is generally not safe to do with esp_hal types, especially those \
    holding buffers for the duration of a data transfer."
)]

extern crate alloc;

use core::net::{IpAddr, Ipv4Addr, SocketAddr};

use alloc::boxed::Box;

use esp_hal::clock::CpuClock;
use esp_hal::interrupt::software::SoftwareInterruptControl;
use esp_hal::rng::Rng;
use esp_hal::timer::timg::TimerGroup;
use esp_hal::Config;

use log::{error, info};

use embassy_executor::Spawner;
use embassy_net::{Config as NetConfig, DhcpConfig, Runner, Stack, StackResources};
use embassy_time::Timer;

use esp_radio::wifi::{
    sta_state, ClientConfig, Config as WifiConfig, ModeConfig, WifiController, WifiDevice,
    WifiEvent, WifiStaState,
};
use esp_radio::Controller;

use edge_http::io::server::{Connection, Handler, Server};
use edge_http::io::Error;
use edge_http::Method;
use edge_nal::TcpBind;
use edge_nal_embassy::{Tcp, TcpBuffers};

use embedded_io_async::{Read, Write};

macro_rules! mk_static {
    ($t:ty,$val:expr) => {{
        static STATIC_CELL: static_cell::StaticCell<$t> = static_cell::StaticCell::new();
        #[deny(unused_attributes)]
        let x = STATIC_CELL.uninit().write($val);
        x
    }};
}

// This creates a default app-descriptor required by the esp-idf bootloader.
// For more information see: <https://docs.espressif.com/projects/esp-idf/en/stable/esp32/api-reference/system/app_image_format.html#application-description>
esp_bootloader_esp_idf::esp_app_desc!();

#[panic_handler]
fn panic(_: &core::panic::PanicInfo) -> ! {
    loop {}
}

#[embassy_executor::task]
async fn connect(mut wifi_controller: WifiController<'static>) {
    info!("Wi-Fi connection task started");
    loop {
        if sta_state() == WifiStaState::Connected {
            wifi_controller
                .wait_for_event(WifiEvent::StaDisconnected)
                .await;
            embassy_time::Timer::after_secs(2).await;
        }

        if !matches!(wifi_controller.is_started(), Ok(true)) {
            info!("Starting Wi-Fi...");
            wifi_controller
                .start_async()
                .await
                .expect("Impossible to start Wi-Fi");
            info!("Wi-Fi started");
        }

        info!("Attempting to connect...");
        if let Err(e) = wifi_controller.connect_async().await {
            error!("Wi-Fi connect failed: {e:?}");
            embassy_time::Timer::after_secs(2).await;
        } else {
            info!("Wi-Fi connected!");
        }
    }
}

#[embassy_executor::task]
async fn task(mut runner: Runner<'static, WifiDevice<'static>>) {
    runner.run().await;
}

#[inline]
pub(crate) async fn get_ip(stack: Stack<'static>) -> Ipv4Addr {
    info!("Waiting till the link is up...");
    loop {
        if stack.is_link_up() {
            break;
        }
        Timer::after_millis(100).await;
    }

    info!("Waiting to get IP address...");
    loop {
        if let Some(config) = stack.config_v4() {
            return config.address.address();
        }
        Timer::after_millis(100).await;
    }
}

#[toml_cfg::toml_config]
struct DeviceConfig {
    #[default("")]
    ssid: &'static str,
    #[default("")]
    password: &'static str,
    #[default("")]
    broker_address: &'static str,
    #[default(0)]
    broker_port: u16,
}

#[esp_rtos::main]
async fn main(spawner: Spawner) {
    esp_println::logger::init_logger_from_env();

    let config = Config::default().with_cpu_clock(CpuClock::max());
    let peripherals = esp_hal::init(config);

    esp_alloc::heap_allocator!(size: 128 * 1024);

    let timg0 = TimerGroup::new(peripherals.TIMG0);
    let sw_int = SoftwareInterruptControl::new(peripherals.SW_INTERRUPT);
    esp_rtos::start(timg0.timer0, sw_int.software_interrupt0);

    info!("ESP RTOS started!");

    // Retrieve device configuration.
    let device_config = DEVICE_CONFIG;

    let esp_radio_controller = &*mk_static!(Controller<'static>, esp_radio::init().unwrap());
    let (mut controller, interfaces) = esp_radio::wifi::new(
        esp_radio_controller,
        peripherals.WIFI,
        WifiConfig::default(),
    )
    .unwrap();

    if device_config.ssid.is_empty() {
        panic!("Missing Wi-Fi SSID");
    }

    if device_config.password.is_empty() {
        panic!("Missing Wi-Fi password");
    }

    let client_config = ModeConfig::Client(
        ClientConfig::default()
            .with_ssid(device_config.ssid.into())
            .with_password(device_config.password.into()),
    );

    controller.set_config(&client_config).unwrap();

    spawner.spawn(connect(controller)).unwrap();

    // Wait until Wi-Fi is connected.
    while sta_state() != WifiStaState::Connected {
        embassy_time::Timer::after_millis(100).await;
    }

    info!("Starting network stack...");

    let rng = Rng::new();

    let net_config = NetConfig::dhcpv4(DhcpConfig::default());
    let seed = u64::from(rng.random()) << 32 | u64::from(rng.random());

    let resources = Box::leak(Box::new(StackResources::<12>::new()));

    let (stack, runner) = embassy_net::new(interfaces.sta, net_config, resources, seed);

    spawner.spawn(task(runner)).unwrap();

    // Wait until the stack has a valid IP configuration.
    while !stack.is_config_up() {
        Timer::after_millis(100).await;
    }

    let mut server = Server::<2, 4096, 32>::new();
    let buffers = TcpBuffers::<2, 2048, 4096>::new();
    let tcp = Tcp::new(stack, &buffers);

    let ip = get_ip(stack).await;

    info!("Ip: {ip}:3000");

    let acceptor = tcp
        .bind(SocketAddr::new(IpAddr::V4(ip), 3000))
        .await
        .unwrap();

    loop {
        match server
            .run(
                Some(15 * 1000),
                edge_nal::WithTimeout::new(15_000, &acceptor),
                HttpHandler,
            )
            .await
        {
            Ok(_) => {}
            Err(error) => {
                // panic!("{:?}", error);
                log::error!("{:?}", error);
            }
        }
    }
}

struct HttpHandler;

impl Handler for HttpHandler {
    type Error<E>
        = Error<E>
    where
        E: core::fmt::Debug;

    async fn handle<T, const N: usize>(
        &self,
        _task_id: impl core::fmt::Display + Copy,
        connection: &mut Connection<'_, T, N>,
    ) -> Result<(), Self::Error<T::Error>>
    where
        T: Read + Write,
    {
        info!("Got new connection");
        let headers = connection.headers()?;

        if headers.method != Method::Get {
            connection
                .initiate_response(405, Some("Method Not Allowed"), &[])
                .await?;
        } else if headers.path != "/" {
            connection
                .initiate_response(404, Some("Not Found"), &[])
                .await?;
        } else {
            connection
                .initiate_response(200, Some("OK"), &[("Content-Type", "text/plain")])
                .await?;

            connection.write_all(b"Hello world!").await?;
        }

        Ok(())
    }
}

build.rs

#[toml_cfg::toml_config]
pub struct DeviceConfig {
    #[default("")]
    ssid: &'static str,
    #[default("")]
    password: &'static str,
}

fn main() {
    let cfg = std::path::Path::new("cfg.toml");

    // Checks whether device configuration exists
    assert!(
        cfg.exists(),
        "A `cfg.toml` file with Wi-Fi credentials and broker configuration is required! Use `cfg.toml.example` as a template."
    );

    let device_config = DEVICE_CONFIG;
    assert!(
        !device_config.ssid.trim().is_empty() || !device_config.password.trim().is_empty(),
        "All config fields should be set in `cfg.toml` file!"
    );

    linker_be_nice();
    // make sure linkall.x is the last linker script (otherwise might cause problems with flip-link)
    println!("cargo:rustc-link-arg=-Tlinkall.x");
}

fn linker_be_nice() {
    let args: Vec<String> = std::env::args().collect();
    if args.len() > 1 {
        let kind = &args[1];
        let what = &args[2];

        match kind.as_str() {
            "undefined-symbol" => match what.as_str() {
                "_defmt_timestamp" => {
                    eprintln!();
                    eprintln!(
                        "💡 `defmt` not found - make sure `defmt.x` is added as a linker script and you have included `use defmt_rtt as _;`"
                    );
                    eprintln!();
                }
                "_stack_start" => {
                    eprintln!();
                    eprintln!("💡 Is the linker script `linkall.x` missing?");
                    eprintln!();
                }
                "esp_rtos_initialized" | "esp_rtos_yield_task" | "esp_rtos_task_create" => {
                    eprintln!();
                    eprintln!(
                        "💡 `esp-radio` has no scheduler enabled. Make sure you have initialized `esp-rtos` or provided an external scheduler."
                    );
                    eprintln!();
                }
                "embedded_test_linker_file_not_added_to_rustflags" => {
                    eprintln!();
                    eprintln!(
                        "💡 `embedded-test` not found - make sure `embedded-test.x` is added as a linker script for tests"
                    );
                    eprintln!();
                }
                _ => (),
            },
            // we don't have anything helpful for "missing-lib" yet
            _ => {
                std::process::exit(1);
            }
        }

        std::process::exit(0);
    }

    println!(
        "cargo:rustc-link-arg=--error-handling-script={}",
        std::env::current_exe().unwrap().display()
    );
}

[ivmarkov]

@copilot please analyze for a potential bug causing this.

NOTE:

• I don't think [edge-http]: Server doesn't seem to free handler after [FIN, ACK] from client. #62 is related. [edge-http]: Server doesn't seem to free handler after [FIN, ACK] from client. #62 is now fixed in the master branch
• I think this has something to do with how we accept incoming TCP connections using embassy-net / smoltcp. don't want to bias your analysis too much (could be something else!) but following is worth pursuing:
  • Typical TCP stacks have a (relatively large) TCP accept queue
  • Typical HTTP servers have a smaller amount of HTTP workers than the TCP accept queue
  • When there are more incoming connections than HTTP workers, the accept queue just gets filled with incoming TCP connection requests, until the workers get free and can handle these; but I think the TCP stack does ACK the incoming connection TCP packet when placing it in the accept queue
  • With smoltcp / embassy-net
    • There is no accept queue
    • If there is no available worker, the incoming connection is not "taken" / "accepted"
  • What you need to do:
    • Examine smoltcp and embassy-net internals, to figure out what happens with a TCP packet when there is nothing to accept it as all workers are busy

[ivmarkov]

@copilot

The work in #87 mitigated this issue to some extent, but if you look at the comments between you and me towards the bottom of the PR, the 1:1 relation between "number of embassy-net+smoltcp sockets" : "number of HTTP tasks" remains.

Please create a new PR which is decoupling this relation and which (assuming the number embassy-net+smoltcp sockets being (much) larger than the number of HTTP tasks) allows for sockets for incoming connections to continue to be accepted even if currently all HTTP tasks are busy processing.

Once an HTTP task is ready / completed (and its socket is closed) it should be able to take a socket, which already accepted an incoming connection and work on that socket.

In a way, we would be emulating the queue of accepted TCP sockets which does not really exist under smoltcp / embassy-net.

[ivmarkov]

@Luni-4 There is now a fix in master (#87) which would alleviate the problem, but not make it completely disappear.

There is also #88, but I'm not sure I'll merge it, because for it to operate correctly, you need to reserve A LOT of sockets (much more than the 2-4 HTTP handlers).

The real fix should be done in smoltcp, and should be along the lines of smoltcp-rs/smoltcp#1120

===

If that's bringing any comfort (likely not) it seems picoserve suffers from the same issue. Which is to be expected, given that the problem is deep down in smoltcp itself.

Actually - and if you are OK with that - you can use a patched version of smoltcp which has the current change surface of smoltcp-rs/smoltcp#1120.

That should be enough to make your current issue disappear.

[Luni-4]

@ivmarkov

Thank you for your hard work!

Over the next few days, I’ll try using master to see whether it’s sufficient for my current use case. For now, I would not go ahead with #88 too, as it appears to require a significant amount of resources.

Could you clarify how I can use a patched smoltcp version in my example? Wouldn't this require using a patched version of edge-net as well?

[ivmarkov]

Over the next few days, I’ll try using master to see whether it’s sufficient for my current use case. For now, I would not go ahead with #88 too, as it appears to require a significant amount of resources.

Well , you have #88 merged now, because it did not require any changes to existing code and is on top of everything else.
But it still sucks, yeah. But we now at least have what picoserve also has, I guess.

Could you clarify how I can use a patched smoltcp version in my example? Wouldn't this require using a patched version of edge-net as well?

Not at all. You can just [patch.crates-io] to use my smoltcp fork here: https://github.com/sysgrok/smoltcp/

Just pin it with "rev" to the current hash (b902c3b3b788ccc00374ad30d81672044f20e334) as I'll likely continue developing it in a backwards incompatible way in the next days/week.

Or just do your own smoltcp fork, apply my 4 lines change there and use yours.

[Luni-4]

Well , you have #88 merged now, because it did not require any changes to existing code and is on top of everything else. But it still sucks, yeah. But we now at least have what picoserve also has, I guess.

Ah ok, it had not been merged when I replied and I thought you did not want to pursue that approach. On second thought, though, it's seems like a good compromise to me, better to have that than nothing.

Not at all. You can just [patch.crates-io] to use my smoltcp fork here: https://github.com/sysgrok/smoltcp/

Just pin it with "rev" to the current hash (b902c3b3b788ccc00374ad30d81672044f20e334) as I'll likely continue developing it in a backwards incompatible way in the next days/week.

Perfect, it's fine with me that way, thanks for your clarification!

[Luni-4]

@ivmarkov

In the meantime, can we publish a new edge-net version with the latest master changes?