diff --git a/app/allowed-licenses.json b/app/allowed-licenses.json index 88ed8ba4d5..9b5ef66556 100644 --- a/app/allowed-licenses.json +++ b/app/allowed-licenses.json @@ -32,6 +32,14 @@ "moduleName": ".*", "moduleLicense": "BSD-4 License" }, + { + "moduleName": ".*", + "moduleLicense": "Revised BSD" + }, + { + "moduleName": ".*", + "moduleLicense": "ISC" + }, { "moduleName": ".*", "moduleLicense": "MIT" @@ -48,6 +56,10 @@ "moduleName": ".*", "moduleLicense": "MIT-0" }, + { + "moduleName": ".*", + "moduleLicense": "MIT license" + }, { "moduleName": "com.github.jai-imageio:jai-imageio-core", "moduleLicense": "LICENSE.txt" diff --git a/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java b/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java index ffbacfee1b..99c8f65117 100644 --- a/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java +++ b/app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java @@ -251,6 +251,21 @@ public class ApplicationProperties { */ private boolean allowPrivateS3Endpoints = false; + /** + * Whether a network source's host (SFTP, FTP, or SMB) may resolve to a loopback, + * link-local, or private address. Off by default so a connection cannot be pointed at + * internal services; enable for an on-network file server (e.g. an internal SFTP drop or a + * Samba share). + */ + private boolean allowPrivateNetworkSources = false; + + /** + * Hostnames (exact, case-insensitive) that a network source may use even when they resolve + * to a private or local address and {@code allowPrivateNetworkSources} is off. Lets shared + * infra allow one named on-prem file server without opening every internal host. + */ + private List allowedPrivateNetworkHosts = new java.util.ArrayList<>(); + /** * Whether an API/Purview/ConsignO integration's base URL may resolve to a loopback, * link-local, or private address. Off by default: unlike S3 connections, any user may diff --git a/app/proprietary/build.gradle b/app/proprietary/build.gradle index 495a013c03..c20241dbb5 100644 --- a/app/proprietary/build.gradle +++ b/app/proprietary/build.gradle @@ -67,6 +67,12 @@ dependencies { implementation "software.amazon.awssdk:s3:${awsSdkVersion}" implementation "software.amazon.awssdk:url-connection-client:${awsSdkVersion}" + // Network policy sources (SFTP/FTP/SMB drop folders). All permissively licensed: + // jsch fork BSD, commons-net Apache-2.0, smbj Apache-2.0 (BouncyCastle already on classpath). + implementation "com.github.mwiede:jsch:${jschVersion}" + implementation "commons-net:commons-net:${commonsNetVersion}" + implementation "com.hierynomus:smbj:${smbjVersion}" + // Streaming AEAD (AES-GCM-HKDF segments) for storage encryption at rest. Apache-2.0. implementation "com.google.crypto.tink:tink:${tinkVersion}" diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/integration/model/IntegrationType.java b/app/proprietary/src/main/java/stirling/software/proprietary/integration/model/IntegrationType.java index 375b7f4604..e372336763 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/integration/model/IntegrationType.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/integration/model/IntegrationType.java @@ -3,6 +3,8 @@ package stirling.software.proprietary.integration.model; /** Kind of external integration a stored config describes. */ public enum IntegrationType { S3, + /** A network file server (SFTP, FTP/FTPS, or SMB/Samba) a policy source polls for documents. */ + NETWORK, MCP, /** A generic outbound HTTP endpoint a pipeline step can post a document to. */ API, diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/integration/service/IntegrationConfigService.java b/app/proprietary/src/main/java/stirling/software/proprietary/integration/service/IntegrationConfigService.java index a6650569a8..a29ac2c08f 100644 --- a/app/proprietary/src/main/java/stirling/software/proprietary/integration/service/IntegrationConfigService.java +++ b/app/proprietary/src/main/java/stirling/software/proprietary/integration/service/IntegrationConfigService.java @@ -57,12 +57,15 @@ public class IntegrationConfigService { OwnerScope scope = request.scope() == null ? OwnerScope.USER : request.scope(); IntegrationConfig cfg = new IntegrationConfig(); cfg.setIntegrationType(require(request.integrationType(), "integrationType")); - // S3 is infrastructure, not self-serve: no personal S3 for regular users. TEAM/SERVER - // scopes are already restricted to admins/team owners by assignOwnership. - if (cfg.getIntegrationType() == IntegrationType.S3 + // S3 and network file servers are infrastructure, not self-serve: they hold shared host + // credentials, so no personal connection for regular users. TEAM/SERVER scopes are already + // restricted to admins/team owners by assignOwnership. + if (isInfrastructureType(cfg.getIntegrationType()) && scope == OwnerScope.USER && !ownership.isAdmin(currentUser)) { - throw forbidden("S3 connections can only be created by administrators or team owners"); + throw forbidden( + "S3 and network connections can only be created by administrators or team" + + " owners"); } requireCustomApiAllowed(cfg.getIntegrationType(), currentUser); cfg.setName(require(request.name(), "name")); @@ -147,6 +150,11 @@ public class IntegrationConfigService { } } + /** Types holding shared host credentials, restricted to admins/team owners like S3. */ + private static boolean isInfrastructureType(IntegrationType type) { + return type == IntegrationType.S3 || type == IntegrationType.NETWORK; + } + /** Whether this caller may author custom API integrations, for the UI to offer or hide it. */ public boolean canAuthorCustomApi(User currentUser) { return applicationProperties.getPolicies().isAllowCustomApiIntegrations() diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/FtpFileClient.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/FtpFileClient.java new file mode 100644 index 0000000000..d493831437 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/FtpFileClient.java @@ -0,0 +1,181 @@ +package stirling.software.proprietary.policy.network; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.net.ftp.FTP; +import org.apache.commons.net.ftp.FTPClient; +import org.apache.commons.net.ftp.FTPFile; +import org.apache.commons.net.ftp.FTPReply; +import org.apache.commons.net.ftp.FTPSClient; + +/** + * FTP client over Apache Commons Net, with optional TLS ({@code EXPLICIT} = AUTH TLS, {@code + * IMPLICIT} = TLS on connect). Passive mode is the default since it works through NAT. A blank + * directory lists the login working directory. One transfer is in flight at a time, so the read + * stream calls {@code completePendingCommand} on close before the session is reused or shut down. + */ +final class FtpFileClient implements RemoteFileClient { + + private static final int CONNECT_TIMEOUT_MS = 15_000; + private static final int READ_TIMEOUT_MS = 60_000; + private static final int MAX_DEPTH = 64; + + private final FTPClient ftp; + + private FtpFileClient(FTPClient ftp) { + this.ftp = ftp; + } + + static FtpFileClient connect(NetworkConfig config) throws IOException { + FTPClient ftp = clientFor(config); + ftp.setConnectTimeout(CONNECT_TIMEOUT_MS); + // Read timeouts so a server that connects then stalls cannot hang a poller: default + // becomes the control socket's SO timeout at connect, data covers transfer sockets. + ftp.setDefaultTimeout(READ_TIMEOUT_MS); + ftp.setDataTimeout(Duration.ofMillis(READ_TIMEOUT_MS)); + try { + ftp.connect(config.host(), config.port()); + int reply = ftp.getReplyCode(); + if (!FTPReply.isPositiveCompletion(reply)) { + ftp.disconnect(); + throw new IOException( + "FTP server " + config.host() + " refused connection: " + reply); + } + if (ftp instanceof FTPSClient ftps) { + // Encrypt the data channel too, not only the control channel. + ftps.execPBSZ(0); + ftps.execPROT("P"); + } + if (!ftp.login(config.username(), config.password())) { + int code = ftp.getReplyCode(); + ftp.disconnect(); + throw new IOException("FTP login to " + config.host() + " failed: " + code); + } + if (config.passive()) { + ftp.enterLocalPassiveMode(); + } + ftp.setFileType(FTP.BINARY_FILE_TYPE); + return new FtpFileClient(ftp); + } catch (IOException e) { + quietlyDisconnect(ftp); + throw e; + } + } + + private static FTPClient clientFor(NetworkConfig config) { + return switch (config.security()) { + case NONE -> new FTPClient(); + case IMPLICIT -> new FTPSClient(true); + case EXPLICIT -> new FTPSClient(false); + }; + } + + @Override + public List list(String directory, boolean recursive) throws IOException { + List files = new ArrayList<>(); + collect(dir(directory), recursive, 0, files); + return files; + } + + private void collect(String directory, boolean recursive, int depth, List out) + throws IOException { + FTPFile[] entries = ftp.listFiles(directory); + for (FTPFile entry : entries) { + if (out.size() >= MAX_FILES) { + return; + } + if (entry == null) { + continue; + } + String name = entry.getName(); + if (name.equals(".") || name.equals("..") || name.startsWith(".")) { + continue; + } + String path = join(directory, name); + if (entry.isDirectory()) { + if (recursive && !entry.isSymbolicLink() && depth < MAX_DEPTH) { + collect(path, true, depth + 1, out); + } + continue; + } + if (entry.isFile()) { + out.add(new RemoteFile(path, name, entry.getSize(), lastModified(entry))); + } + } + } + + @Override + public RemoteFile stat(String path) throws IOException { + // LIST, not MLST: listing uses listFiles, so the re-stat that guards the consume delete + // must read the timestamp the same way or the version gate never matches (and many servers + // do not implement MLST at all, returning null and skipping the delete entirely). + FTPFile[] listed = ftp.listFiles(path); + if (listed.length == 0 || listed[0] == null) { + return null; + } + FTPFile entry = listed[0]; + String name = path.substring(path.lastIndexOf('/') + 1); + return new RemoteFile(path, name, entry.getSize(), lastModified(entry)); + } + + @Override + public InputStream open(String path) throws IOException { + InputStream stream = ftp.retrieveFileStream(path); + if (stream == null) { + throw new IOException("cannot read " + path + ": " + ftp.getReplyString()); + } + // The data stream must be closed and the transfer acknowledged before the session is + // reused. + return new FilterInputStream(stream) { + @Override + public void close() throws IOException { + super.close(); + ftp.completePendingCommand(); + } + }; + } + + @Override + public void delete(String path) throws IOException { + ftp.deleteFile(path); + } + + @Override + public void close() throws IOException { + try { + ftp.logout(); + } finally { + quietlyDisconnect(ftp); + } + } + + private static long lastModified(FTPFile entry) { + return entry.getTimestamp() == null ? 0L : entry.getTimestamp().getTimeInMillis(); + } + + private static void quietlyDisconnect(FTPClient ftp) { + if (ftp.isConnected()) { + try { + ftp.disconnect(); + } catch (IOException ignored) { + // Best-effort teardown; the session is being abandoned anyway. + } + } + } + + private static String dir(String directory) { + return directory == null || directory.isBlank() ? "." : directory; + } + + private static String join(String directory, String name) { + if (directory.equals(".")) { + return name; + } + return directory.endsWith("/") ? directory + name : directory + "/" + name; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/NetworkConfig.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/NetworkConfig.java new file mode 100644 index 0000000000..403799a1bf --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/NetworkConfig.java @@ -0,0 +1,222 @@ +package stirling.software.proprietary.policy.network; + +import java.util.Map; + +/** + * The fully resolved settings a network source runs with - produced by {@link + * NetworkConnectionResolver} merging a stored connection (protocol, host, credentials, and, for + * SMB, the share) with per-use source options (directory, mode, recursive). Credentials are + * required: there is no anonymous fallback, so a source can never reach a server without + * config-supplied identity. {@code snapshot} and {@code recursive} are input-only. + */ +public record NetworkConfig( + NetworkProtocol protocol, + String host, + int port, + String username, + String password, + String privateKey, + String privateKeyPassphrase, + String hostKeyFingerprint, + String domain, + String share, + FtpSecurity security, + boolean passive, + String directory, + boolean snapshot, + boolean recursive) { + + /** FTP transport security: plaintext, or TLS negotiated up-front (implicit) or via AUTH TLS. */ + public enum FtpSecurity { + NONE, + EXPLICIT, + IMPLICIT; + + static FtpSecurity fromOption(String value) { + if (value == null || value.isBlank()) { + return NONE; + } + String normalized = value.trim().toUpperCase(); + for (FtpSecurity security : values()) { + if (security.name().equals(normalized)) { + return security; + } + } + throw new IllegalArgumentException( + "network config 'security' must be NONE, EXPLICIT or IMPLICIT"); + } + } + + private static final String PROTOCOL_OPTION = "protocol"; + private static final String HOST_OPTION = "host"; + private static final String PORT_OPTION = "port"; + private static final String USERNAME_OPTION = "username"; + private static final String PASSWORD_OPTION = "password"; + private static final String PRIVATE_KEY_OPTION = "privateKey"; + private static final String PRIVATE_KEY_PASSPHRASE_OPTION = "privateKeyPassphrase"; + private static final String HOST_KEY_FINGERPRINT_OPTION = "hostKeyFingerprint"; + private static final String DOMAIN_OPTION = "domain"; + private static final String SHARE_OPTION = "share"; + private static final String SECURITY_OPTION = "security"; + private static final String PASSIVE_OPTION = "passive"; + private static final String DIRECTORY_OPTION = "directory"; + private static final String MODE_OPTION = "mode"; + private static final String MODE_CONSUME = "consume"; + private static final String MODE_SNAPSHOT = "snapshot"; + private static final String RECURSIVE_OPTION = "recursive"; + + public static NetworkConfig from(Map options) { + NetworkProtocol protocol = NetworkProtocol.fromOption(str(options.get(PROTOCOL_OPTION))); + if (protocol == null) { + throw new IllegalArgumentException( + "network config requires a 'protocol' of sftp, ftp or smb"); + } + String host = trimmed(options.get(HOST_OPTION)); + if (host == null) { + throw new IllegalArgumentException("network config requires a 'host'"); + } + FtpSecurity security = FtpSecurity.fromOption(str(options.get(SECURITY_OPTION))); + int port = port(options.get(PORT_OPTION), defaultPort(protocol, security)); + String username = trimmed(options.get(USERNAME_OPTION)); + if (username == null) { + throw new IllegalArgumentException("network config requires a 'username'"); + } + String password = trimmed(options.get(PASSWORD_OPTION)); + String privateKey = trimmed(options.get(PRIVATE_KEY_OPTION)); + String passphrase = trimmed(options.get(PRIVATE_KEY_PASSPHRASE_OPTION)); + String fingerprint = trimmed(options.get(HOST_KEY_FINGERPRINT_OPTION)); + String domain = trimmed(options.get(DOMAIN_OPTION)); + String share = trimmed(options.get(SHARE_OPTION)); + boolean passive = parseBoolean(options.get(PASSIVE_OPTION), true); + String directory = directory(options.get(DIRECTORY_OPTION)); + boolean recursive = parseBoolean(options.get(RECURSIVE_OPTION), false); + boolean snapshot = snapshot(str(options.get(MODE_OPTION))); + + if (protocol == NetworkProtocol.SFTP && password == null && privateKey == null) { + throw new IllegalArgumentException( + "sftp connection requires a 'password' or a 'privateKey'"); + } + if ((protocol == NetworkProtocol.FTP || protocol == NetworkProtocol.SMB) + && password == null) { + throw new IllegalArgumentException( + protocol.name() + " connection requires a 'password'"); + } + if (protocol == NetworkProtocol.SMB && share == null) { + throw new IllegalArgumentException( + "smb connection requires a 'share' (e.g. documents)"); + } + return new NetworkConfig( + protocol, + host, + port, + username, + password, + privateKey, + passphrase, + fingerprint, + domain, + share, + security, + passive, + directory, + snapshot, + recursive); + } + + /** Implicit FTPS listens on 990, not the plain-FTP 21. */ + private static int defaultPort(NetworkProtocol protocol, FtpSecurity security) { + if (protocol == NetworkProtocol.FTP && security == FtpSecurity.IMPLICIT) { + return 990; + } + return protocol.defaultPort(); + } + + private static int port(Object value, int fallback) { + String text = trimmed(value); + if (text == null) { + return fallback; + } + int port; + try { + port = Integer.parseInt(text); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("network config 'port' must be a number"); + } + if (port < 1 || port > 65535) { + throw new IllegalArgumentException("network config 'port' must be between 1 and 65535"); + } + return port; + } + + /** + * A poll directory relative to the login/home dir (SFTP/FTP) or the share root (SMB). Traversal + * segments are rejected so a connection cannot be steered above where the operator scoped it. + */ + private static String directory(Object value) { + String text = trimmed(value); + if (text == null) { + return ""; + } + String normalized = text.replace('\\', '/'); + if (normalized.indexOf('\0') >= 0) { + throw new IllegalArgumentException("network config 'directory' contains a null byte"); + } + for (String segment : normalized.split("/")) { + if (segment.equals("..")) { + throw new IllegalArgumentException( + "network config 'directory' must not contain '..'"); + } + } + return normalized; + } + + private static boolean snapshot(String mode) { + if (mode == null || mode.isBlank()) { + return false; + } + if (!MODE_CONSUME.equals(mode) && !MODE_SNAPSHOT.equals(mode)) { + throw new IllegalArgumentException( + "network config 'mode' must be 'consume' or 'snapshot'"); + } + return MODE_SNAPSHOT.equals(mode); + } + + private static boolean parseBoolean(Object value, boolean fallback) { + String text = trimmed(value); + return text == null ? fallback : Boolean.parseBoolean(text); + } + + private static String str(Object value) { + return value == null ? null : value.toString(); + } + + private static String trimmed(Object value) { + if (value == null) { + return null; + } + String text = value.toString().trim(); + return text.isEmpty() ? null : text; + } + + /** Never prints the credentials, so an accidental log line cannot leak them. */ + @Override + public String toString() { + return "NetworkConfig[protocol=" + + protocol + + ", host=" + + host + + ", port=" + + port + + ", username=" + + username + + ", share=" + + share + + ", directory=" + + directory + + ", snapshot=" + + snapshot + + ", recursive=" + + recursive + + "]"; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/NetworkConnectionResolver.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/NetworkConnectionResolver.java new file mode 100644 index 0000000000..098b31b561 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/NetworkConnectionResolver.java @@ -0,0 +1,136 @@ +package stirling.software.proprietary.policy.network; + +import java.util.LinkedHashMap; +import java.util.Map; + +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.security.core.userdetails.UserDetails; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import lombok.RequiredArgsConstructor; + +import stirling.software.proprietary.access.model.ResourceType; +import stirling.software.proprietary.access.service.OwnershipService; +import stirling.software.proprietary.integration.model.IntegrationConfig; +import stirling.software.proprietary.integration.model.IntegrationType; +import stirling.software.proprietary.integration.repository.IntegrationConfigRepository; +import stirling.software.proprietary.security.model.User; +import stirling.software.proprietary.security.service.UserService; + +import tools.jackson.core.type.TypeReference; +import tools.jackson.databind.ObjectMapper; + +/** + * Turns a network source's options into a full {@link NetworkConfig} by dereferencing its {@code + * connectionId} to a stored NETWORK {@link IntegrationConfig} (the connection owns protocol, host, + * credentials and, for SMB, the share; the options own the per-use directory and mode). Options + * with no {@code connectionId} are parsed directly, so a programmatic caller can embed config. + * + *

Mirrors {@code S3ConnectionResolver}: an authenticated caller (save-time validation) must be + * allowed to use the connection; a background sweep with no caller skips that check, since the + * referencing source was access-checked when it was saved. + */ +@Service +@RequiredArgsConstructor +@Transactional(readOnly = true) +public class NetworkConnectionResolver { + + static final String CONNECTION_ID_OPTION = "connectionId"; + private static final String DIRECTORY_OPTION = "directory"; + private static final String MODE_OPTION = "mode"; + private static final String RECURSIVE_OPTION = "recursive"; + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private final IntegrationConfigRepository connections; + private final OwnershipService ownership; + private final UserService userService; + + public NetworkConfig resolve(Map options) { + Long connectionId = connectionId(options); + if (connectionId == null) { + return NetworkConfig.from(options); + } + IntegrationConfig connection = + connections + .findById(connectionId) + .filter(cfg -> cfg.getIntegrationType() == IntegrationType.NETWORK) + .filter(this::usableByCurrentUser) + // Existence and access collapse into one error so a caller cannot tell + // "no such connection" from "someone else's" and enumerate ids. + .orElseThrow( + () -> + new IllegalArgumentException( + "unknown or inaccessible network connection")); + if (!connection.isEnabled()) { + throw new IllegalArgumentException("network connection is disabled"); + } + Map merged = new LinkedHashMap<>(connectionConfig(connection)); + copyPerUseOption(options, merged, DIRECTORY_OPTION); + copyPerUseOption(options, merged, MODE_OPTION); + copyPerUseOption(options, merged, RECURSIVE_OPTION); + return NetworkConfig.from(merged); + } + + static Long connectionId(Map options) { + Object reference = options.get(CONNECTION_ID_OPTION); + if (reference == null || (reference instanceof String s && s.isBlank())) { + return null; + } + if (reference instanceof Number number) { + return number.longValue(); + } + try { + return Long.valueOf(reference.toString().trim()); + } catch (NumberFormatException e) { + throw new IllegalArgumentException( + "network 'connectionId' is not a valid connection reference: " + reference); + } + } + + private boolean usableByCurrentUser(IntegrationConfig connection) { + User user = currentUser(); + return user == null || ownership.canUse(ResourceType.INTEGRATION_CONFIG, connection, user); + } + + private User currentUser() { + Authentication auth = SecurityContextHolder.getContext().getAuthentication(); + if (auth == null || !auth.isAuthenticated()) { + return null; + } + Object principal = auth.getPrincipal(); + if (principal instanceof User user) { + return user; + } + if (principal instanceof UserDetails userDetails) { + return userService.findByUsername(userDetails.getUsername()).orElse(null); + } + if (principal instanceof String username && !"anonymousUser".equals(username)) { + return userService.findByUsername(username).orElse(null); + } + return null; + } + + private static Map connectionConfig(IntegrationConfig connection) { + String json = connection.getConfig(); + if (json == null || json.isBlank()) { + return Map.of(); + } + try { + return OBJECT_MAPPER.readValue( + json, new TypeReference>() {}); + } catch (Exception e) { + throw new IllegalArgumentException( + "network connection '" + connection.getName() + "' has unreadable config", e); + } + } + + private static void copyPerUseOption( + Map options, Map merged, String key) { + Object value = options.get(key); + if (value != null && !value.toString().isBlank()) { + merged.put(key, value); + } + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/NetworkHostGuard.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/NetworkHostGuard.java new file mode 100644 index 0000000000..44a174f989 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/NetworkHostGuard.java @@ -0,0 +1,68 @@ +package stirling.software.proprietary.policy.network; + +import java.net.InetAddress; +import java.net.UnknownHostException; + +import org.springframework.stereotype.Component; + +import lombok.RequiredArgsConstructor; + +import stirling.software.common.model.ApplicationProperties; + +/** + * Refuses a network source whose host resolves to a loopback, link-local, or private address, + * unless the operator opts in via {@code policies.allowPrivateNetworkSources} (every host) or names + * the host in {@code policies.allowedPrivateNetworkHosts} (that host only, for shared infra). The + * host comes from a portal user, so without this a connection could be aimed at internal services + * (the cloud metadata address, an admin panel). Mirrors the S3 endpoint guard; enforced both at + * save time and before every connect. + */ +@Component +@RequiredArgsConstructor +public class NetworkHostGuard { + + private final ApplicationProperties applicationProperties; + + public void requirePermitted(String host) { + if (applicationProperties.getPolicies().isAllowPrivateNetworkSources()) { + return; + } + if (host == null || host.isBlank()) { + throw new IllegalArgumentException("network source requires a host"); + } + if (isAllowlisted(host)) { + return; + } + InetAddress[] addresses; + try { + addresses = InetAddress.getAllByName(host); + } catch (UnknownHostException e) { + throw new IllegalArgumentException("cannot resolve network host '" + host + "'", e); + } + for (InetAddress address : addresses) { + if (isPrivateOrLocal(address)) { + throw new IllegalArgumentException( + "network host '" + + host + + "' resolves to a private or local address (" + + address.getHostAddress() + + "); add it to policies.allowedPrivateNetworkHosts or set" + + " policies.allowPrivateNetworkSources=true to allow an" + + " on-network server"); + } + } + } + + private boolean isAllowlisted(String host) { + return applicationProperties.getPolicies().getAllowedPrivateNetworkHosts().stream() + .anyMatch(allowed -> allowed != null && allowed.trim().equalsIgnoreCase(host)); + } + + private static boolean isPrivateOrLocal(InetAddress address) { + return address.isLoopbackAddress() + || address.isLinkLocalAddress() + || address.isSiteLocalAddress() + || address.isAnyLocalAddress() + || address.isMulticastAddress(); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/NetworkIdentities.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/NetworkIdentities.java new file mode 100644 index 0000000000..fb8d344abd --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/NetworkIdentities.java @@ -0,0 +1,52 @@ +package stirling.software.proprietary.policy.network; + +/** + * The network backend's identity and version scheme, shared by {@link NetworkInputSource} and its + * completion hook. Identity keys a file's ledger row and stays stable across sweeps; the gate is + * size + last-modified, since SFTP/FTP/SMB expose no ETag - the same "stat" strength as a folder + * source, so a touch that changes mtime counts as a new version. + */ +public final class NetworkIdentities { + + private NetworkIdentities() {} + + /** A stable, unique key for a remote file: {@code protocol://host:port[/share]/path}. */ + public static String identity(NetworkConfig config, String path) { + StringBuilder id = + new StringBuilder(config.protocol().name().toLowerCase()) + .append("://") + .append(config.host()) + .append(':') + .append(config.port()); + if (config.share() != null && !config.share().isBlank()) { + id.append('/').append(trimSlashes(config.share())); + } + id.append('/').append(trimLeadingSlash(path)); + return id.toString(); + } + + /** The version gate: size and last-modified, mirroring the folder source's stat identity. */ + public static String gate(long size, long lastModifiedMs) { + return size + ":" + lastModifiedMs; + } + + private static String trimSlashes(String value) { + int start = 0; + int end = value.length(); + while (start < end && (value.charAt(start) == '/' || value.charAt(start) == '\\')) { + start++; + } + while (end > start && (value.charAt(end - 1) == '/' || value.charAt(end - 1) == '\\')) { + end--; + } + return value.substring(start, end); + } + + private static String trimLeadingSlash(String value) { + int start = 0; + while (start < value.length() && value.charAt(start) == '/') { + start++; + } + return value.substring(start); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/NetworkInputSource.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/NetworkInputSource.java new file mode 100644 index 0000000000..0b6ff0aa45 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/NetworkInputSource.java @@ -0,0 +1,148 @@ +package stirling.software.proprietary.policy.network; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.springframework.core.io.Resource; +import org.springframework.stereotype.Service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import stirling.software.proprietary.policy.input.InputSource; +import stirling.software.proprietary.policy.input.ResolveContext; +import stirling.software.proprietary.policy.input.ResolvedInput; +import stirling.software.proprietary.policy.model.InputSpec; +import stirling.software.proprietary.policy.model.PolicyInputs; + +/** + * Reads input files from a network file server (SFTP, FTP/FTPS, or SMB), one bean serving all three + * source types ({@code sftp}, {@code ftp}, {@code network}); the protocol comes from the referenced + * connection. Each listed file is its own unit of work, claimed through the {@link ResolveContext} + * ledger and tracked in place, mirroring the S3 source. The version gate is size + last-modified + * (there is no ETag), so this has folder-source "stat" strength. Options: "connectionId" references + * the stored NETWORK connection; "directory" is the poll folder (relative to the login home or + * share root); "recursive" descends into subdirectories; "mode" is "consume" (default: a processed + * file is deleted once every policy that claimed it has settled successfully and it is still the + * version that ran) or "snapshot" (stateless, every run sees the full set). + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class NetworkInputSource implements InputSource { + + private final NetworkConnectionResolver connectionResolver; + private final RemoteFileClientFactory clientFactory; + + @Override + public String type() { + return NetworkProtocol.SMB.sourceType(); + } + + @Override + public boolean supports(InputSpec spec) { + return spec != null && NetworkProtocol.forSourceType(spec.type()) != null; + } + + /** + * Fails fast at save time: an unknown/disabled/inaccessible connection, a protocol that does + * not match the source type, a blocked host, or a server the connection cannot list. + */ + @Override + public void validate(InputSpec spec) { + NetworkConfig config = connectionResolver.resolve(spec.options()); + NetworkProtocol expected = NetworkProtocol.forSourceType(spec.type()); + if (expected != null && config.protocol() != expected) { + throw new IllegalArgumentException( + "source type '" + + spec.type() + + "' does not match the connection protocol " + + config.protocol()); + } + try (RemoteFileClient client = clientFactory.connect(config)) { + client.list(config.directory(), false); + } catch (IOException e) { + throw new IllegalArgumentException( + "cannot access " + config.protocol() + " source: " + e.getMessage(), e); + } + } + + @Override + public List resolve(InputSpec spec, ResolveContext ctx) throws IOException { + NetworkConfig config = connectionResolver.resolve(spec.options()); + // A listing failure propagates so the sweep reads it as "could not list" (which vetoes + // presence cleanup), never as "verifiably no files". + List files; + try (RemoteFileClient client = clientFactory.connect(config)) { + files = client.list(config.directory(), config.recursive()); + } + if (files.size() >= RemoteFileClient.MAX_FILES) { + log.warn( + "Network source listing hit the {}-file cap; remaining files are picked up" + + " on later sweeps", + RemoteFileClient.MAX_FILES); + } + + if (config.snapshot()) { + return files.stream() + .map(file -> ResolvedInput.of(PolicyInputs.of(List.of(resource(config, file))))) + .toList(); + } + + ctx.reportPresent( + files.stream() + .map(file -> NetworkIdentities.identity(config, file.path())) + .toList()); + + List work = new ArrayList<>(); + for (RemoteFile file : files) { + String identity = NetworkIdentities.identity(config, file.path()); + String gate = NetworkIdentities.gate(file.size(), file.lastModifiedMs()); + if (!ctx.claim(identity, gate, null)) { + continue; + } + work.add( + new ResolvedInput( + PolicyInputs.of(List.of(resource(config, file))), + success -> + completeConsumed(ctx, config, file, identity, gate, success))); + } + return work; + } + + /** + * Settle at the version this run claimed, then remove the file only when it still carries that + * version and every policy that claimed it has settled DONE, mirroring the S3 source's + * consensus delete. A failed run settles ERROR and never deletes; the DONE row of a file that + * could not be deleted still stops reprocessing. + */ + private void completeConsumed( + ResolveContext ctx, + NetworkConfig config, + RemoteFile file, + String identity, + String claimGate, + boolean success) { + ctx.settle(identity, claimGate, null, success); + if (!success) { + return; + } + try (RemoteFileClient client = clientFactory.connect(config)) { + RemoteFile current = client.stat(file.path()); + if (current == null) { + return; // removed by the user or a co-watching policy's consensus delete + } + String currentGate = NetworkIdentities.gate(current.size(), current.lastModifiedMs()); + if (currentGate.equals(claimGate) && ctx.allSettledDone(identity)) { + client.delete(file.path()); + } + } catch (IOException e) { + log.warn("Could not remove consumed network file {}: {}", identity, e.getMessage()); + } + } + + private Resource resource(NetworkConfig config, RemoteFile file) { + return new RemoteFileResource(clientFactory, config, file); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/NetworkIntegrationValidator.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/NetworkIntegrationValidator.java new file mode 100644 index 0000000000..a9ec9cc260 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/NetworkIntegrationValidator.java @@ -0,0 +1,35 @@ +package stirling.software.proprietary.policy.network; + +import java.util.Map; + +import org.springframework.stereotype.Component; + +import lombok.RequiredArgsConstructor; + +import stirling.software.proprietary.integration.model.IntegrationType; +import stirling.software.proprietary.integration.service.IntegrationConfigValidator; + +/** + * The network connection schema, enforced when a NETWORK {@link IntegrationType} config is saved: + * protocol, host, credentials (and, for SMB, a share) required, with the same private-address guard + * {@link RemoteFileClientFactory} applies before connecting, moved to save time so a bad connection + * fails in the form rather than in a sweep. A live connectivity check runs later, when the source + * that uses the connection is saved ({@link NetworkInputSource#validate}). + */ +@Component +@RequiredArgsConstructor +public class NetworkIntegrationValidator implements IntegrationConfigValidator { + + private final NetworkHostGuard hostGuard; + + @Override + public IntegrationType type() { + return IntegrationType.NETWORK; + } + + @Override + public void validate(Map config) { + NetworkConfig parsed = NetworkConfig.from(config); + hostGuard.requirePermitted(parsed.host()); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/NetworkProtocol.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/NetworkProtocol.java new file mode 100644 index 0000000000..bbd6dc5d7e --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/NetworkProtocol.java @@ -0,0 +1,57 @@ +package stirling.software.proprietary.policy.network; + +/** + * The file-transfer protocols a network source speaks. Each maps to a {@link RemoteFileClient} + * implementation and to a portal source type: {@code sftp} -> SFTP, {@code ftp} -> FTP/FTPS, {@code + * network} -> SMB (a Windows/Samba share). The stored connection carries the protocol; the source + * type only routes the UI. + */ +public enum NetworkProtocol { + SFTP(22, "sftp"), + FTP(21, "ftp"), + SMB(445, "network"); + + private final int defaultPort; + private final String sourceType; + + NetworkProtocol(int defaultPort, String sourceType) { + this.defaultPort = defaultPort; + this.sourceType = sourceType; + } + + public int defaultPort() { + return defaultPort; + } + + /** The portal source {@code type} string this protocol backs. */ + public String sourceType() { + return sourceType; + } + + /** Parse a stored {@code protocol} option (case-insensitive), or null when absent/unknown. */ + public static NetworkProtocol fromOption(String value) { + if (value == null) { + return null; + } + String normalized = value.trim().toUpperCase(); + for (NetworkProtocol protocol : values()) { + if (protocol.name().equals(normalized)) { + return protocol; + } + } + return null; + } + + /** The protocol a source {@code type} string implies, or null when it is not a network type. */ + public static NetworkProtocol forSourceType(String type) { + if (type == null) { + return null; + } + for (NetworkProtocol protocol : values()) { + if (protocol.sourceType.equals(type)) { + return protocol; + } + } + return null; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/RemoteFile.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/RemoteFile.java new file mode 100644 index 0000000000..8c72ddab64 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/RemoteFile.java @@ -0,0 +1,18 @@ +package stirling.software.proprietary.policy.network; + +/** + * One remote file discovered by a {@link RemoteFileClient} listing: its full path on the server, + * its display name, size in bytes, and last-modified epoch milliseconds. Size and mtime form the + * version gate ({@link NetworkIdentities#gate}) since network protocols expose no ETag. + */ +public record RemoteFile(String path, String name, long size, long lastModifiedMs) { + + public RemoteFile { + if (path == null || path.isBlank()) { + throw new IllegalArgumentException("remote file requires a path"); + } + if (name == null || name.isBlank()) { + name = path.substring(path.lastIndexOf('/') + 1); + } + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/RemoteFileClient.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/RemoteFileClient.java new file mode 100644 index 0000000000..217c590031 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/RemoteFileClient.java @@ -0,0 +1,38 @@ +package stirling.software.proprietary.policy.network; + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +/** + * A connected session to one network file server (SFTP, FTP, or SMB), created by {@link + * RemoteFileClientFactory}. Sessions are short-lived and single-use: the caller opens one, performs + * a listing, a read, or a delete, then closes it - there is no pooling, because network sessions + * time out and go stale far more readily than an S3 HTTP client. Callers must {@link #close()} + * every client (try-with-resources), except when streaming, where ownership is handed to the + * returned stream so it can be read after {@code resolve} returns. + */ +public interface RemoteFileClient extends Closeable { + + /** + * Listing cap: a listing stops once this many files are collected, bounding memory against a + * huge (or maliciously deep) tree. Consume mode drains the rest on later sweeps. + */ + int MAX_FILES = 10_000; + + /** + * Every regular, non-hidden file under {@code directory}, optionally descending into it, + * truncated at {@link #MAX_FILES}. + */ + List list(String directory, boolean recursive) throws IOException; + + /** The current metadata for one path, or null when it no longer exists. */ + RemoteFile stat(String path) throws IOException; + + /** Opens the file for reading; the stream stays valid until this client is closed. */ + InputStream open(String path) throws IOException; + + /** Removes the file; a no-op if it is already gone. */ + void delete(String path) throws IOException; +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/RemoteFileClientFactory.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/RemoteFileClientFactory.java new file mode 100644 index 0000000000..fe3f43b1bb --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/RemoteFileClientFactory.java @@ -0,0 +1,28 @@ +package stirling.software.proprietary.policy.network; + +import java.io.IOException; + +import org.springframework.stereotype.Component; + +import lombok.RequiredArgsConstructor; + +/** + * Opens a fresh {@link RemoteFileClient} for one {@link NetworkConfig}, dispatched by protocol. The + * host is guarded against private addresses before every connect, since it comes from a portal user + * and each operation opens its own short-lived session (there is no long-lived pool to guard once). + */ +@Component +@RequiredArgsConstructor +public class RemoteFileClientFactory { + + private final NetworkHostGuard hostGuard; + + public RemoteFileClient connect(NetworkConfig config) throws IOException { + hostGuard.requirePermitted(config.host()); + return switch (config.protocol()) { + case SFTP -> SftpFileClient.connect(config); + case FTP -> FtpFileClient.connect(config); + case SMB -> SmbFileClient.connect(config); + }; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/RemoteFileResource.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/RemoteFileResource.java new file mode 100644 index 0000000000..e9429e0ea7 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/RemoteFileResource.java @@ -0,0 +1,68 @@ +package stirling.software.proprietary.policy.network; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; + +import org.springframework.core.io.AbstractResource; + +/** + * A policy input backed by a remote file, streamed on demand. Each read opens its own short-lived + * client (the listing client is long gone by the time the run reads the file) and hands it to the + * stream, so closing the stream tears the session down. Size and name come from the listing, so the + * pipeline can size the input without a second round trip. + */ +final class RemoteFileResource extends AbstractResource { + + private final RemoteFileClientFactory factory; + private final NetworkConfig config; + private final RemoteFile file; + + RemoteFileResource(RemoteFileClientFactory factory, NetworkConfig config, RemoteFile file) { + this.factory = factory; + this.config = config; + this.file = file; + } + + @Override + public InputStream getInputStream() throws IOException { + RemoteFileClient client = factory.connect(config); + try { + InputStream stream = client.open(file.path()); + return new FilterInputStream(stream) { + @Override + public void close() throws IOException { + try { + super.close(); + } finally { + client.close(); + } + } + }; + } catch (IOException e) { + client.close(); + throw e; + } + } + + /** Listed just now; a reader gets a precise error from {@link #getInputStream} instead. */ + @Override + public boolean exists() { + return true; + } + + @Override + public long contentLength() { + return file.size(); + } + + @Override + public String getFilename() { + return file.name(); + } + + @Override + public String getDescription() { + return "network file " + NetworkIdentities.identity(config, file.path()); + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/SftpFileClient.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/SftpFileClient.java new file mode 100644 index 0000000000..5ecc29392d --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/SftpFileClient.java @@ -0,0 +1,312 @@ +package stirling.software.proprietary.policy.network; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.Locale; +import java.util.Vector; + +import com.jcraft.jsch.ChannelSftp; +import com.jcraft.jsch.HostKey; +import com.jcraft.jsch.HostKeyRepository; +import com.jcraft.jsch.JSch; +import com.jcraft.jsch.JSchException; +import com.jcraft.jsch.Session; +import com.jcraft.jsch.SftpATTRS; +import com.jcraft.jsch.SftpException; +import com.jcraft.jsch.UserInfo; + +import stirling.software.common.configuration.InstallationPathConfig; + +/** + * SFTP client over jsch. Host keys are verified: a connection with a configured {@code + * hostKeyFingerprint} only accepts the server presenting that key; otherwise the key seen on the + * first connect is pinned in a known-hosts file under the config directory and any later change is + * refused (trust-on-first-use). Directories are listed, read and deleted through one channel; a + * blank directory means the login home. Symlinked directories are not followed and hidden entries + * are skipped, mirroring the folder source. + */ +final class SftpFileClient implements RemoteFileClient { + + private static final int CONNECT_TIMEOUT_MS = 15_000; + private static final int READ_TIMEOUT_MS = 60_000; + private static final int MAX_DEPTH = 64; + static final String KNOWN_HOSTS_FILE = "sftp_known_hosts"; + + private final Session session; + private final ChannelSftp channel; + + private SftpFileClient(Session session, ChannelSftp channel) { + this.session = session; + this.channel = channel; + } + + static SftpFileClient connect(NetworkConfig config) throws IOException { + JSch jsch = new JSch(); + try { + if (config.privateKey() != null) { + byte[] passphrase = + config.privateKeyPassphrase() == null + ? null + : config.privateKeyPassphrase().getBytes(StandardCharsets.UTF_8); + jsch.addIdentity( + "network-source", + config.privateKey().getBytes(StandardCharsets.UTF_8), + null, + passphrase); + } + Session session = jsch.getSession(config.username(), config.host(), config.port()); + if (config.password() != null) { + session.setPassword(config.password()); + } + if (config.hostKeyFingerprint() != null) { + // Pinned key: only the configured fingerprint is ever accepted. + session.setHostKeyRepository( + new PinnedHostKeyRepository(config.hostKeyFingerprint())); + } else { + // Trust-on-first-use: the first key seen is recorded, a changed key is refused. + jsch.setKnownHosts(knownHostsFile().toString()); + session.setHostKeyRepository( + new TofuHostKeyRepository(jsch.getHostKeyRepository())); + } + session.setConfig("StrictHostKeyChecking", "yes"); + session.connect(CONNECT_TIMEOUT_MS); + session.setTimeout(READ_TIMEOUT_MS); + ChannelSftp channel = (ChannelSftp) session.openChannel("sftp"); + channel.connect(CONNECT_TIMEOUT_MS); + return new SftpFileClient(session, channel); + } catch (JSchException e) { + throw new IOException( + "SFTP connection to " + config.host() + " failed: " + e.getMessage(), e); + } + } + + /** jsch only persists newly pinned keys into a file that already exists, so create it. */ + private static Path knownHostsFile() throws IOException { + Path file = Path.of(InstallationPathConfig.getConfigPath(), KNOWN_HOSTS_FILE); + Files.createDirectories(file.getParent()); + try { + Files.createFile(file); + } catch (FileAlreadyExistsException ignored) { + // Another connect already created it; the content is managed by jsch. + } + return file; + } + + /** + * Trust-on-first-use over the shared known-hosts file: an unknown host's key is pinned on first + * contact, and a key that later differs is refused ({@code CHANGED}). Rejection happens before + * authentication, so credentials are never sent to an impersonating server. + */ + private static final class TofuHostKeyRepository implements HostKeyRepository { + + private final HostKeyRepository knownHosts; + + private TofuHostKeyRepository(HostKeyRepository knownHosts) { + this.knownHosts = knownHosts; + } + + @Override + public int check(String host, byte[] key) { + int result = knownHosts.check(host, key); + if (result != NOT_INCLUDED) { + return result; + } + try { + knownHosts.add(new HostKey(host, key), null); + } catch (JSchException e) { + return CHANGED; // a key we cannot even parse is refused, not trusted + } + return OK; + } + + @Override + public void add(HostKey hostkey, UserInfo ui) { + knownHosts.add(hostkey, ui); + } + + @Override + public void remove(String host, String type) { + knownHosts.remove(host, type); + } + + @Override + public void remove(String host, String type, byte[] key) { + knownHosts.remove(host, type, key); + } + + @Override + public String getKnownHostsRepositoryID() { + return knownHosts.getKnownHostsRepositoryID(); + } + + @Override + public HostKey[] getHostKey() { + return knownHosts.getHostKey(); + } + + @Override + public HostKey[] getHostKey(String host, String type) { + return knownHosts.getHostKey(host, type); + } + } + + /** + * Accepts only the server key whose SHA-256 fingerprint matches the configured value (OpenSSH + * {@code SHA256:base64} form, prefix optional). Rejection happens before authentication, so + * credentials are never sent to an impersonating server. + */ + private static final class PinnedHostKeyRepository implements HostKeyRepository { + + private final String expected; + + private PinnedHostKeyRepository(String fingerprint) { + this.expected = normalize(fingerprint); + } + + @Override + public int check(String host, byte[] key) { + return expected.equals(sha256Fingerprint(key)) ? OK : CHANGED; + } + + private static String normalize(String fingerprint) { + String value = fingerprint.trim(); + if (value.toUpperCase(Locale.ROOT).startsWith("SHA256:")) { + value = value.substring("SHA256:".length()); + } + // OpenSSH prints the digest base64 without padding; accept it padded too. + return value.replace("=", ""); + } + + private static String sha256Fingerprint(byte[] key) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + return Base64.getEncoder().withoutPadding().encodeToString(digest.digest(key)); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 unavailable", e); + } + } + + @Override + public void add(HostKey hostkey, UserInfo ui) {} + + @Override + public void remove(String host, String type) {} + + @Override + public void remove(String host, String type, byte[] key) {} + + @Override + public String getKnownHostsRepositoryID() { + return "pinned-fingerprint"; + } + + @Override + public HostKey[] getHostKey() { + return new HostKey[0]; + } + + @Override + public HostKey[] getHostKey(String host, String type) { + return new HostKey[0]; + } + } + + @Override + public List list(String directory, boolean recursive) throws IOException { + List files = new ArrayList<>(); + collect(dir(directory), recursive, 0, files); + return files; + } + + @SuppressWarnings("unchecked") + private void collect(String directory, boolean recursive, int depth, List out) + throws IOException { + Vector entries; + try { + entries = channel.ls(directory); + } catch (SftpException e) { + throw new IOException("cannot list " + directory + ": " + e.getMessage(), e); + } + for (ChannelSftp.LsEntry entry : entries) { + if (out.size() >= MAX_FILES) { + return; + } + String name = entry.getFilename(); + if (name.equals(".") || name.equals("..") || name.startsWith(".")) { + continue; + } + SftpATTRS attrs = entry.getAttrs(); + String path = join(directory, name); + if (attrs.isDir()) { + if (recursive && !attrs.isLink() && depth < MAX_DEPTH) { + collect(path, true, depth + 1, out); + } + continue; + } + if (attrs.isReg()) { + out.add(new RemoteFile(path, name, attrs.getSize(), attrs.getMTime() * 1000L)); + } + } + } + + @Override + public RemoteFile stat(String path) throws IOException { + try { + SftpATTRS attrs = channel.stat(path); + String name = path.substring(path.lastIndexOf('/') + 1); + return new RemoteFile(path, name, attrs.getSize(), attrs.getMTime() * 1000L); + } catch (SftpException e) { + if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) { + return null; + } + throw new IOException("cannot stat " + path + ": " + e.getMessage(), e); + } + } + + @Override + public InputStream open(String path) throws IOException { + try { + return channel.get(path); + } catch (SftpException e) { + throw new IOException("cannot read " + path + ": " + e.getMessage(), e); + } + } + + @Override + public void delete(String path) throws IOException { + try { + channel.rm(path); + } catch (SftpException e) { + if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) { + return; + } + throw new IOException("cannot delete " + path + ": " + e.getMessage(), e); + } + } + + @Override + public void close() { + channel.disconnect(); + session.disconnect(); + } + + private static String dir(String directory) { + return directory == null || directory.isBlank() ? "." : directory; + } + + private static String join(String directory, String name) { + if (directory.equals(".")) { + return name; + } + return directory.endsWith("/") ? directory + name : directory + "/" + name; + } +} diff --git a/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/SmbFileClient.java b/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/SmbFileClient.java new file mode 100644 index 0000000000..b49d6dad53 --- /dev/null +++ b/app/proprietary/src/main/java/stirling/software/proprietary/policy/network/SmbFileClient.java @@ -0,0 +1,236 @@ +package stirling.software.proprietary.policy.network; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import com.hierynomus.msdtyp.AccessMask; +import com.hierynomus.mserref.NtStatus; +import com.hierynomus.msfscc.FileAttributes; +import com.hierynomus.msfscc.fileinformation.FileAllInformation; +import com.hierynomus.msfscc.fileinformation.FileIdBothDirectoryInformation; +import com.hierynomus.mssmb2.SMB2CreateDisposition; +import com.hierynomus.mssmb2.SMB2ShareAccess; +import com.hierynomus.mssmb2.SMBApiException; +import com.hierynomus.protocol.commons.socket.ProxySocketFactory; +import com.hierynomus.smbj.SMBClient; +import com.hierynomus.smbj.SmbConfig; +import com.hierynomus.smbj.auth.AuthenticationContext; +import com.hierynomus.smbj.common.SMBRuntimeException; +import com.hierynomus.smbj.connection.Connection; +import com.hierynomus.smbj.session.Session; +import com.hierynomus.smbj.share.DiskShare; + +/** + * SMB2/3 client over smbj against a Windows or Samba share. The share is named on the connection; + * directories are relative to its root, using backslashes as SMB expects. A blank directory lists + * the share root. Hidden and system entries are skipped, and symlink/reparse dirs are not + * descended, mirroring the folder source. + */ +final class SmbFileClient implements RemoteFileClient { + + private static final int CONNECT_TIMEOUT_MS = 15_000; + private static final int READ_TIMEOUT_MS = 60_000; + private static final int MAX_DEPTH = 64; + + private final SMBClient smbClient; + private final Connection connection; + private final Session session; + private final DiskShare share; + + private SmbFileClient( + SMBClient smbClient, Connection connection, Session session, DiskShare share) { + this.smbClient = smbClient; + this.connection = connection; + this.session = session; + this.share = share; + } + + static SmbFileClient connect(NetworkConfig config) throws IOException { + // Bounded connect and read timeouts (both default to unlimited) so a host that accepts + // the TCP connection then stalls cannot wedge a poller. + SmbConfig smbConfig = + SmbConfig.builder() + .withSocketFactory(new ProxySocketFactory(CONNECT_TIMEOUT_MS)) + .withSoTimeout(READ_TIMEOUT_MS, TimeUnit.MILLISECONDS) + .withTimeout(READ_TIMEOUT_MS, TimeUnit.MILLISECONDS) + .build(); + SMBClient smbClient = new SMBClient(smbConfig); + Connection connection = smbClient.connect(config.host(), config.port()); + try { + char[] password = + config.password() == null ? new char[0] : config.password().toCharArray(); + AuthenticationContext auth = + new AuthenticationContext(config.username(), password, config.domain()); + Session session = connection.authenticate(auth); + DiskShare share = (DiskShare) session.connectShare(config.share()); + return new SmbFileClient(smbClient, connection, session, share); + } catch (RuntimeException e) { + closeQuietly(connection); + closeQuietly(smbClient); + throw new IOException( + "SMB connection to \\\\" + + config.host() + + "\\" + + config.share() + + " failed: " + + e.getMessage(), + e); + } + } + + @Override + public List list(String directory, boolean recursive) throws IOException { + List files = new ArrayList<>(); + collect(smbDir(directory), recursive, 0, files); + return files; + } + + private void collect(String directory, boolean recursive, int depth, List out) + throws IOException { + List entries; + try { + entries = share.list(directory); + } catch (SMBRuntimeException e) { + // smbj throws unchecked; surface as IOException like the SFTP/FTP clients. + throw new IOException("cannot list " + slashed(directory) + ": " + e.getMessage(), e); + } + for (FileIdBothDirectoryInformation info : entries) { + if (out.size() >= MAX_FILES) { + return; + } + String name = info.getFileName(); + if (name.equals(".") || name.equals("..") || name.startsWith(".")) { + continue; + } + long attributes = info.getFileAttributes(); + if (isSet(attributes, FileAttributes.FILE_ATTRIBUTE_HIDDEN) + || isSet(attributes, FileAttributes.FILE_ATTRIBUTE_SYSTEM)) { + continue; + } + String path = join(directory, name); + if (isSet(attributes, FileAttributes.FILE_ATTRIBUTE_DIRECTORY)) { + if (recursive + && !isSet(attributes, FileAttributes.FILE_ATTRIBUTE_REPARSE_POINT) + && depth < MAX_DEPTH) { + collect(path, true, depth + 1, out); + } + continue; + } + out.add( + new RemoteFile( + slashed(path), + name, + info.getEndOfFile(), + info.getLastWriteTime().toEpochMillis())); + } + } + + @Override + public RemoteFile stat(String path) throws IOException { + String smbPath = smbPath(path); + try { + FileAllInformation info = share.getFileInformation(smbPath); + String name = path.substring(path.lastIndexOf('/') + 1); + return new RemoteFile( + slashed(smbPath), + name, + info.getStandardInformation().getEndOfFile(), + info.getBasicInformation().getLastWriteTime().toEpochMillis()); + } catch (SMBApiException e) { + if (isNotFound(e)) { + return null; + } + throw new IOException("cannot stat " + path + ": " + e.getMessage(), e); + } + } + + @Override + public InputStream open(String path) throws IOException { + try { + com.hierynomus.smbj.share.File file = + share.openFile( + smbPath(path), + EnumSet.of(AccessMask.GENERIC_READ), + null, + SMB2ShareAccess.ALL, + SMB2CreateDisposition.FILE_OPEN, + null); + // Close the SMB file handle when the stream is closed, before the session is torn down. + return new FilterInputStream(file.getInputStream()) { + @Override + public void close() throws IOException { + try { + super.close(); + } finally { + file.close(); + } + } + }; + } catch (SMBApiException e) { + throw new IOException("cannot read " + path + ": " + e.getMessage(), e); + } + } + + @Override + public void delete(String path) throws IOException { + try { + share.rm(smbPath(path)); + } catch (SMBApiException e) { + if (isNotFound(e)) { + return; + } + throw new IOException("cannot delete " + path + ": " + e.getMessage(), e); + } + } + + @Override + public void close() { + closeQuietly(share); + closeQuietly(session); + closeQuietly(connection); + closeQuietly(smbClient); + } + + private static boolean isNotFound(SMBApiException e) { + return e.getStatus() == NtStatus.STATUS_OBJECT_NAME_NOT_FOUND + || e.getStatus() == NtStatus.STATUS_OBJECT_PATH_NOT_FOUND; + } + + private static boolean isSet(long attributes, FileAttributes attribute) { + return (attributes & attribute.getValue()) != 0; + } + + private static void closeQuietly(AutoCloseable closeable) { + try { + closeable.close(); + } catch (Exception ignored) { + // Best-effort teardown; the session is being abandoned anyway. + } + } + + /** The share-relative listing path: blank means the share root, which smbj lists as "". */ + private static String smbDir(String directory) { + return directory == null || directory.isBlank() ? "" : smbPath(directory); + } + + /** SMB paths use backslashes; internal identities keep forward slashes. */ + private static String smbPath(String path) { + return path.replace('/', '\\'); + } + + private static String slashed(String path) { + return path.replace('\\', '/'); + } + + private static String join(String directory, String name) { + if (directory.isEmpty()) { + return name; + } + return directory.endsWith("\\") ? directory + name : directory + "\\" + name; + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/network/FtpNetworkSourceIntegrationTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/network/FtpNetworkSourceIntegrationTest.java new file mode 100644 index 0000000000..567ed4e7cf --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/network/FtpNetworkSourceIntegrationTest.java @@ -0,0 +1,199 @@ +package stirling.software.proprietary.policy.network; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.FixedHostPortGenericContainer; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.junit.jupiter.Testcontainers; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.proprietary.access.service.OwnershipService; +import stirling.software.proprietary.integration.repository.IntegrationConfigRepository; +import stirling.software.proprietary.policy.input.ResolveContext; +import stirling.software.proprietary.policy.input.ResolvedInput; +import stirling.software.proprietary.policy.ledger.InProcessProcessedLedger; +import stirling.software.proprietary.policy.model.InputSpec; +import stirling.software.proprietary.security.service.UserService; + +/** + * End-to-end {@link NetworkInputSource} test over FTP against a real vsftpd server, through the + * production {@link FtpFileClient}: listing, claiming, streaming the actual bytes (exercising the + * passive data channel and {@code completePendingCommand}), consume delete, and save-time + * validation. Passive FTP hands the client a data port, so the control and one passive port are + * pinned to fixed host ports and the server advertises 127.0.0.1 - the only way this works through + * Testcontainers' per-run port mapping. + */ +@Testcontainers(disabledWithoutDocker = true) +@EnabledIfEnvironmentVariable( + named = "RUN_NETWORK_INTEGRATION_TESTS", + matches = "true", + disabledReason = + "Spins up SFTP/FTP/SMB containers; opt-in to keep several heavy containers off the" + + " standard CI runner. Run with RUN_NETWORK_INTEGRATION_TESTS=true.") +class FtpNetworkSourceIntegrationTest { + + private static final String POLICY = "p1"; + private static final String USER = "stirling"; + private static final String PASS = "secret"; + private static final String HOME = "/ftp/stirling"; + // Passive FTP needs the data port pinned to a known host port, so pick two free ones up front + // (rather than hard-coding) to avoid clashing with anything else on a CI runner. + private static final int CONTROL_PORT; + private static final int PASSIVE_PORT; + + static { + try (java.net.ServerSocket control = new java.net.ServerSocket(0); + java.net.ServerSocket passive = new java.net.ServerSocket(0)) { + CONTROL_PORT = control.getLocalPort(); + PASSIVE_PORT = passive.getLocalPort(); + } catch (java.io.IOException e) { + throw new java.io.UncheckedIOException(e); + } + } + + @org.testcontainers.junit.jupiter.Container + static GenericContainer ftp = + new FixedHostPortGenericContainer<>("delfer/alpine-ftp-server:latest") + .withFixedExposedPort(CONTROL_PORT, 21) + .withFixedExposedPort(PASSIVE_PORT, PASSIVE_PORT) + .withEnv("USERS", USER + "|" + PASS + "|" + HOME + "|1000") + .withEnv("ADDRESS", "127.0.0.1") + .withEnv("MIN_PORT", String.valueOf(PASSIVE_PORT)) + .withEnv("MAX_PORT", String.valueOf(PASSIVE_PORT)) + .waitingFor(Wait.forListeningPort()); + + private NetworkInputSource source; + private InProcessProcessedLedger ledger; + private RecordingContext ctx; + + @BeforeEach + void setUp() { + ApplicationProperties properties = new ApplicationProperties(); + properties.getPolicies().setAllowPrivateNetworkSources(true); + RemoteFileClientFactory factory = + new RemoteFileClientFactory(new NetworkHostGuard(properties)); + NetworkConnectionResolver resolver = + new NetworkConnectionResolver( + mock(IntegrationConfigRepository.class), + mock(OwnershipService.class), + mock(UserService.class)); + source = new NetworkInputSource(resolver, factory); + ledger = new InProcessProcessedLedger(); + ctx = new RecordingContext(); + } + + @Test + void consumeListsStreamsAndDeletes() throws Exception { + put("doc.pdf", "hello ftp"); + + List work = source.resolve(spec(Map.of()), ctx); + + assertThat(work).hasSize(1); + assertThat(read(work.get(0))).isEqualTo("hello ftp"); + assertThat(source.resolve(spec(Map.of()), ctx)).isEmpty(); + + work.get(0).onComplete().accept(true); + assertThat(exists("doc.pdf")).isFalse(); + assertThat(source.resolve(spec(Map.of()), ctx)).isEmpty(); + } + + @Test + void aFailedFileStaysOnTheServer() throws Exception { + put("doc.pdf", "data"); + + source.resolve(spec(Map.of()), ctx).get(0).onComplete().accept(false); + + assertThat(exists("doc.pdf")).isTrue(); + assertThat(source.resolve(spec(Map.of()), ctx)).isEmpty(); + } + + @Test + void validateConnectsAndRejectsBadCredentials() { + source.validate(spec(Map.of())); + + Map wrong = new HashMap<>(baseOptions()); + wrong.put("password", "not-the-password"); + assertThatThrownBy(() -> source.validate(new InputSpec("ftp", wrong))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("cannot access"); + } + + private Map baseOptions() { + Map options = new HashMap<>(); + options.put("protocol", "ftp"); + options.put("host", "127.0.0.1"); + options.put("port", String.valueOf(CONTROL_PORT)); + options.put("username", USER); + options.put("password", PASS); + return options; + } + + private InputSpec spec(Map extra) { + Map options = new HashMap<>(baseOptions()); + options.putAll(extra); + return new InputSpec("ftp", options); + } + + private void put(String name, String content) throws Exception { + String path = HOME + "/" + name; + exec("printf '%s' '" + content + "' > " + path + " && chown 1000 " + path); + } + + private boolean exists(String name) throws Exception { + return exec("test -f " + HOME + "/" + name + " && echo yes || echo no").contains("yes"); + } + + private String exec(String script) throws Exception { + Container.ExecResult result = ftp.execInContainer("sh", "-c", script); + return result.getStdout() + result.getStderr(); + } + + private static String read(ResolvedInput unit) throws IOException { + try (InputStream stream = unit.inputs().primary().get(0).getInputStream()) { + return new String(stream.readAllBytes(), StandardCharsets.UTF_8); + } + } + + private class RecordingContext implements ResolveContext { + + private final List present = new ArrayList<>(); + + @Override + public boolean claim(String identity, String gate, Supplier contentHash) { + return ledger.claim(POLICY, identity, gate, contentHash); + } + + @Override + public void settle( + String identity, String finalGate, String finalContentHash, boolean success) { + ledger.settle(POLICY, identity, finalGate, finalContentHash, success); + } + + @Override + public boolean allSettledDone(String identity) { + return ledger.allSettledDone(identity); + } + + @Override + public void reportPresent(Collection identities) { + present.addAll(identities); + } + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/network/NetworkConfigTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/network/NetworkConfigTest.java new file mode 100644 index 0000000000..3fb48501f5 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/network/NetworkConfigTest.java @@ -0,0 +1,136 @@ +package stirling.software.proprietary.policy.network; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.LinkedHashMap; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +class NetworkConfigTest { + + private static Map base(String protocol) { + Map options = new LinkedHashMap<>(); + options.put("protocol", protocol); + options.put("host", "files.example.com"); + options.put("username", "u"); + options.put("password", "p"); + return options; + } + + @Test + void parsesAnSftpConnectionWithDefaultPort() { + NetworkConfig config = NetworkConfig.from(base("sftp")); + assertEquals(NetworkProtocol.SFTP, config.protocol()); + assertEquals(22, config.port()); + assertEquals("files.example.com", config.host()); + } + + @Test + void defaultsPortPerProtocol() { + assertEquals(21, NetworkConfig.from(base("ftp")).port()); + Map smb = base("smb"); + smb.put("share", "docs"); + assertEquals(445, NetworkConfig.from(smb).port()); + } + + @Test + void implicitFtpsDefaultsToPort990() { + Map implicit = base("ftp"); + implicit.put("security", "implicit"); + assertEquals(990, NetworkConfig.from(implicit).port()); + + implicit.put("port", "2121"); + assertEquals(2121, NetworkConfig.from(implicit).port()); + } + + @Test + void sftpAcceptsAPrivateKeyInsteadOfAPassword() { + Map options = base("sftp"); + options.remove("password"); + options.put("privateKey", "dummy-test-key-material"); + NetworkConfig config = NetworkConfig.from(options); + assertEquals(NetworkProtocol.SFTP, config.protocol()); + } + + @Test + void sftpWithoutPasswordOrKeyIsRejected() { + Map options = base("sftp"); + options.remove("password"); + assertThrows(IllegalArgumentException.class, () -> NetworkConfig.from(options)); + } + + @Test + void ftpRequiresAPassword() { + Map options = base("ftp"); + options.remove("password"); + assertThrows(IllegalArgumentException.class, () -> NetworkConfig.from(options)); + } + + @Test + void smbRequiresAShare() { + assertThrows(IllegalArgumentException.class, () -> NetworkConfig.from(base("smb"))); + } + + @Test + void requiresProtocolHostAndUsername() { + assertThrows( + IllegalArgumentException.class, + () -> NetworkConfig.from(Map.of("host", "h", "username", "u", "password", "p"))); + Map noHost = base("sftp"); + noHost.remove("host"); + assertThrows(IllegalArgumentException.class, () -> NetworkConfig.from(noHost)); + Map noUser = base("sftp"); + noUser.remove("username"); + assertThrows(IllegalArgumentException.class, () -> NetworkConfig.from(noUser)); + } + + @Test + void rejectsAnOutOfRangePort() { + Map options = base("sftp"); + options.put("port", "70000"); + assertThrows(IllegalArgumentException.class, () -> NetworkConfig.from(options)); + } + + @Test + void rejectsATraversalDirectory() { + Map options = base("sftp"); + options.put("directory", "in/../../etc"); + assertThrows(IllegalArgumentException.class, () -> NetworkConfig.from(options)); + } + + @Test + void mapsModeToSnapshotFlag() { + Map consume = base("sftp"); + consume.put("mode", "consume"); + assertFalse(NetworkConfig.from(consume).snapshot()); + + Map snapshot = base("sftp"); + snapshot.put("mode", "snapshot"); + assertTrue(NetworkConfig.from(snapshot).snapshot()); + + Map bad = base("sftp"); + bad.put("mode", "sometimes"); + assertThrows(IllegalArgumentException.class, () -> NetworkConfig.from(bad)); + } + + @Test + void carriesTheHostKeyFingerprint() { + Map options = base("sftp"); + options.put("hostKeyFingerprint", " SHA256:abc123 "); + assertEquals("SHA256:abc123", NetworkConfig.from(options).hostKeyFingerprint()); + assertNull(NetworkConfig.from(base("sftp")).hostKeyFingerprint()); + } + + @Test + void toStringHidesSecrets() { + Map options = base("sftp"); + options.put("password", "hunter2"); + String text = NetworkConfig.from(options).toString(); + assertFalse(text.contains("hunter2"), "password must not appear in toString"); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/network/NetworkHostGuardTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/network/NetworkHostGuardTest.java new file mode 100644 index 0000000000..fffbf77305 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/network/NetworkHostGuardTest.java @@ -0,0 +1,64 @@ +package stirling.software.proprietary.policy.network; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import stirling.software.common.model.ApplicationProperties; + +class NetworkHostGuardTest { + + private static NetworkHostGuard guard(boolean allowPrivate) { + ApplicationProperties properties = new ApplicationProperties(); + properties.getPolicies().setAllowPrivateNetworkSources(allowPrivate); + return new NetworkHostGuard(properties); + } + + private static NetworkHostGuard guardWithAllowlist(String... hosts) { + ApplicationProperties properties = new ApplicationProperties(); + properties.getPolicies().setAllowedPrivateNetworkHosts(List.of(hosts)); + return new NetworkHostGuard(properties); + } + + @Test + void rejectsLoopbackByDefault() { + assertThrows( + IllegalArgumentException.class, () -> guard(false).requirePermitted("127.0.0.1")); + } + + @Test + void rejectsPrivateRangesByDefault() { + assertThrows( + IllegalArgumentException.class, () -> guard(false).requirePermitted("10.0.0.1")); + assertThrows( + IllegalArgumentException.class, + () -> guard(false).requirePermitted("192.168.1.10")); + } + + @Test + void rejectsABlankHost() { + assertThrows(IllegalArgumentException.class, () -> guard(false).requirePermitted("")); + } + + @Test + void optInAllowsPrivateAddresses() { + assertDoesNotThrow(() -> guard(true).requirePermitted("127.0.0.1")); + assertDoesNotThrow(() -> guard(true).requirePermitted("10.0.0.1")); + } + + @Test + void allowlistedHostIsAllowedWithoutTheGlobalFlag() { + assertDoesNotThrow(() -> guardWithAllowlist("10.0.0.1").requirePermitted("10.0.0.1")); + assertDoesNotThrow(() -> guardWithAllowlist("FILES.local").requirePermitted("files.LOCAL")); + } + + @Test + void allowlistDoesNotOpenOtherPrivateHosts() { + assertThrows( + IllegalArgumentException.class, + () -> guardWithAllowlist("10.0.0.1").requirePermitted("10.0.0.2")); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/network/NetworkIdentitiesTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/network/NetworkIdentitiesTest.java new file mode 100644 index 0000000000..00f3637bdf --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/network/NetworkIdentitiesTest.java @@ -0,0 +1,49 @@ +package stirling.software.proprietary.policy.network; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.LinkedHashMap; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +class NetworkIdentitiesTest { + + private static NetworkConfig config(String protocol, String share) { + Map options = new LinkedHashMap<>(); + options.put("protocol", protocol); + options.put("host", "files.example.com"); + options.put("port", "2222"); + options.put("username", "u"); + options.put("password", "p"); + if (share != null) { + options.put("share", share); + } + return NetworkConfig.from(options); + } + + @Test + void identityIsProtocolHostPortAndPath() { + assertEquals( + "sftp://files.example.com:2222/in/doc.pdf", + NetworkIdentities.identity(config("sftp", null), "in/doc.pdf")); + } + + @Test + void smbIdentityIncludesTheShare() { + Map options = new LinkedHashMap<>(); + options.put("protocol", "smb"); + options.put("host", "files.example.com"); + options.put("username", "u"); + options.put("password", "p"); + options.put("share", "documents"); + assertEquals( + "smb://files.example.com:445/documents/reports/q1.pdf", + NetworkIdentities.identity(NetworkConfig.from(options), "reports/q1.pdf")); + } + + @Test + void gateIsSizeAndModifiedTime() { + assertEquals("42:1700000000000", NetworkIdentities.gate(42, 1700000000000L)); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/network/NetworkInputSourceTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/network/NetworkInputSourceTest.java new file mode 100644 index 0000000000..c125dd4919 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/network/NetworkInputSourceTest.java @@ -0,0 +1,280 @@ +package stirling.software.proprietary.policy.network; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.proprietary.access.service.OwnershipService; +import stirling.software.proprietary.integration.repository.IntegrationConfigRepository; +import stirling.software.proprietary.policy.input.ResolveContext; +import stirling.software.proprietary.policy.input.ResolvedInput; +import stirling.software.proprietary.policy.ledger.InProcessProcessedLedger; +import stirling.software.proprietary.policy.model.InputSpec; +import stirling.software.proprietary.security.service.UserService; + +/** + * Tests for {@link NetworkInputSource}: consume mode tracks remote files through the ledger, the + * version-guarded delete survives a mid-run replacement, a shared file is removed only once every + * policy is done, snapshot stays stateless, and the source type must match the connection protocol. + * The remote server is faked in memory, so no real SFTP/FTP/SMB endpoint is needed. + */ +class NetworkInputSourceTest { + + private static final String POLICY = "p1"; + + private FakeServer server; + private NetworkInputSource source; + private InProcessProcessedLedger ledger; + private RecordingContext ctx; + + @BeforeEach + void setUp() { + server = new FakeServer(); + // allowPrivate=true so the host guard never resolves DNS; connect() is overridden anyway. + ApplicationProperties properties = new ApplicationProperties(); + properties.getPolicies().setAllowPrivateNetworkSources(true); + RemoteFileClientFactory factory = + new RemoteFileClientFactory(new NetworkHostGuard(properties)) { + @Override + public RemoteFileClient connect(NetworkConfig config) { + return new FakeClient(server); + } + }; + NetworkConnectionResolver resolver = + new NetworkConnectionResolver( + mock(IntegrationConfigRepository.class), + mock(OwnershipService.class), + mock(UserService.class)); + source = new NetworkInputSource(resolver, factory); + ledger = new InProcessProcessedLedger(); + ctx = new RecordingContext(); + } + + private static InputSpec sftp(Map extra) { + Map options = new LinkedHashMap<>(); + options.put("protocol", "sftp"); + options.put("host", "files.example.com"); + options.put("username", "u"); + options.put("password", "p"); + options.put("directory", "in"); + options.putAll(extra); + return new InputSpec("sftp", options); + } + + @Test + void supportsTheThreeNetworkSourceTypes() { + assertTrue(source.supports(new InputSpec("sftp", Map.of()))); + assertTrue(source.supports(new InputSpec("ftp", Map.of()))); + assertTrue(source.supports(new InputSpec("network", Map.of()))); + assertFalse(source.supports(new InputSpec("folder", Map.of()))); + } + + @Test + void consumeRemovesTheFileOnceProcessed() throws Exception { + server.put("in/doc.pdf", "data", 1000); + + List work = source.resolve(sftp(Map.of()), ctx); + + assertEquals(1, work.size()); + assertEquals(1, work.get(0).inputs().primary().size()); + // In flight: still on the server, but a second sweep does not pick it up again. + assertTrue(server.has("in/doc.pdf")); + assertTrue(source.resolve(sftp(Map.of()), ctx).isEmpty()); + + work.get(0).onComplete().accept(true); + assertFalse(server.has("in/doc.pdf")); + assertTrue(source.resolve(sftp(Map.of()), ctx).isEmpty()); + } + + @Test + void aFileReplacedMidRunSurvivesTheDeleteAndRunsAgain() throws Exception { + server.put("in/doc.pdf", "data", 1000); + + List work = source.resolve(sftp(Map.of()), ctx); + // A new version lands while the run is executing (different size/mtime = new gate). + server.put("in/doc.pdf", "new data, different size", 2000); + work.get(0).onComplete().accept(true); + + // The delete is version-guarded: the replacement is not the file that ran, so it stays and + // is claimed as fresh work next sweep. + assertTrue(server.has("in/doc.pdf")); + assertEquals(1, source.resolve(sftp(Map.of()), ctx).size()); + } + + @Test + void aSharedFileIsRemovedOnlyOnceEveryPolicyHasProcessedIt() throws Exception { + server.put("in/doc.pdf", "data", 1000); + RecordingContext other = new RecordingContext("p2"); + + List mine = source.resolve(sftp(Map.of()), ctx); + List theirs = source.resolve(sftp(Map.of()), other); + assertEquals(1, mine.size()); + assertEquals(1, theirs.size()); + + mine.get(0).onComplete().accept(true); + assertTrue(server.has("in/doc.pdf")); // the other claim is still in flight + + theirs.get(0).onComplete().accept(true); + assertFalse(server.has("in/doc.pdf")); + } + + @Test + void aFailedFileStaysInPlaceAndIsNotRetriedUntilItChanges() throws Exception { + server.put("in/doc.pdf", "data", 1000); + + source.resolve(sftp(Map.of()), ctx).get(0).onComplete().accept(false); + + assertTrue(server.has("in/doc.pdf")); + assertTrue(source.resolve(sftp(Map.of()), ctx).isEmpty()); + + server.put("in/doc.pdf", "data", 2000); // touched: new mtime is a new version + assertEquals(1, source.resolve(sftp(Map.of()), ctx).size()); + } + + @Test + void snapshotReadsStatelesslyEverySweep() throws Exception { + server.put("in/doc.pdf", "data", 1000); + InputSpec spec = sftp(Map.of("mode", "snapshot")); + + List first = source.resolve(spec, ctx); + first.get(0).onComplete().accept(true); + List second = source.resolve(spec, ctx); + + assertEquals(1, first.size()); + assertEquals(1, second.size()); // no ledger involvement + assertTrue(ctx.present.isEmpty()); + assertTrue(server.has("in/doc.pdf")); + } + + @Test + void streamsTheFileContent() throws Exception { + server.put("in/doc.pdf", "hello", 1000); + + List work = source.resolve(sftp(Map.of()), ctx); + try (InputStream in = work.get(0).inputs().primary().get(0).getInputStream()) { + assertEquals("hello", new String(in.readAllBytes(), StandardCharsets.UTF_8)); + } + } + + @Test + void validateRejectsASourceTypeThatDoesNotMatchTheConnectionProtocol() { + // A "network" (SMB) source pointed at an SFTP connection is a misconfiguration. + Map options = new LinkedHashMap<>(); + options.put("protocol", "sftp"); + options.put("host", "files.example.com"); + options.put("username", "u"); + options.put("password", "p"); + assertThrows( + IllegalArgumentException.class, + () -> source.validate(new InputSpec("network", options))); + } + + /** In-memory remote server shared between a test's fake clients. */ + private static final class FakeServer { + private final Map files = new LinkedHashMap<>(); + + void put(String path, String content, long mtime) { + files.put(path, new FakeFile(content.getBytes(StandardCharsets.UTF_8), mtime)); + } + + boolean has(String path) { + return files.containsKey(path); + } + } + + private record FakeFile(byte[] content, long mtime) {} + + /** A RemoteFileClient over the in-memory server; every op reads/writes the shared map. */ + private static final class FakeClient implements RemoteFileClient { + private final FakeServer server; + + private FakeClient(FakeServer server) { + this.server = server; + } + + @Override + public List list(String directory, boolean recursive) { + List out = new ArrayList<>(); + server.files.forEach( + (path, file) -> { + String name = path.substring(path.lastIndexOf('/') + 1); + out.add(new RemoteFile(path, name, file.content().length, file.mtime())); + }); + return out; + } + + @Override + public RemoteFile stat(String path) { + FakeFile file = server.files.get(path); + if (file == null) { + return null; + } + String name = path.substring(path.lastIndexOf('/') + 1); + return new RemoteFile(path, name, file.content().length, file.mtime()); + } + + @Override + public InputStream open(String path) { + return new ByteArrayInputStream(server.files.get(path).content()); + } + + @Override + public void delete(String path) { + server.files.remove(path); + } + + @Override + public void close() {} + } + + /** Policy-scoped context backed by the in-process ledger, recording presence reports. */ + private final class RecordingContext implements ResolveContext { + private final String policyId; + private final List present = new ArrayList<>(); + + private RecordingContext() { + this(POLICY); + } + + private RecordingContext(String policyId) { + this.policyId = policyId; + } + + @Override + public boolean claim(String identity, String gate, Supplier contentHash) { + return ledger.claim(policyId, identity, gate, contentHash); + } + + @Override + public void settle( + String identity, String finalGate, String finalContentHash, boolean success) { + ledger.settle(policyId, identity, finalGate, finalContentHash, success); + } + + @Override + public boolean allSettledDone(String identity) { + return ledger.allSettledDone(identity); + } + + @Override + public void reportPresent(Collection identities) { + present.addAll(identities); + } + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/network/NetworkIntegrationValidatorTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/network/NetworkIntegrationValidatorTest.java new file mode 100644 index 0000000000..40a3fb70a8 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/network/NetworkIntegrationValidatorTest.java @@ -0,0 +1,55 @@ +package stirling.software.proprietary.policy.network; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.LinkedHashMap; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.proprietary.integration.model.IntegrationType; + +class NetworkIntegrationValidatorTest { + + private static NetworkIntegrationValidator validator(boolean allowPrivate) { + ApplicationProperties properties = new ApplicationProperties(); + properties.getPolicies().setAllowPrivateNetworkSources(allowPrivate); + return new NetworkIntegrationValidator(new NetworkHostGuard(properties)); + } + + private static Map config(String host) { + Map options = new LinkedHashMap<>(); + options.put("protocol", "sftp"); + options.put("host", host); + options.put("username", "u"); + options.put("password", "p"); + return options; + } + + @Test + void reportsTheNetworkType() { + assertEquals(IntegrationType.NETWORK, validator(true).type()); + } + + @Test + void acceptsAValidConfig() { + assertDoesNotThrow(() -> validator(true).validate(config("10.0.0.5"))); + } + + @Test + void rejectsAMalformedConfig() { + assertThrows( + IllegalArgumentException.class, + () -> validator(true).validate(Map.of("host", "h"))); + } + + @Test + void rejectsAPrivateHostUnlessOptedIn() { + assertThrows( + IllegalArgumentException.class, + () -> validator(false).validate(config("10.0.0.5"))); + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/network/SftpNetworkSourceIntegrationTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/network/SftpNetworkSourceIntegrationTest.java new file mode 100644 index 0000000000..3df4f49cb3 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/network/SftpNetworkSourceIntegrationTest.java @@ -0,0 +1,241 @@ +package stirling.software.proprietary.policy.network; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.junit.jupiter.Testcontainers; + +import stirling.software.common.configuration.InstallationPathConfig; +import stirling.software.common.model.ApplicationProperties; +import stirling.software.proprietary.access.service.OwnershipService; +import stirling.software.proprietary.integration.repository.IntegrationConfigRepository; +import stirling.software.proprietary.policy.input.ResolveContext; +import stirling.software.proprietary.policy.input.ResolvedInput; +import stirling.software.proprietary.policy.ledger.InProcessProcessedLedger; +import stirling.software.proprietary.policy.model.InputSpec; +import stirling.software.proprietary.security.service.UserService; + +/** + * End-to-end {@link NetworkInputSource} test over SFTP against a real atmoz/sftp server, through + * the production {@link SftpFileClient}: listing, claiming, streaming the actual bytes, consume + * delete, and save-time validation. The server chroots the user to its home; {@code upload} is the + * writable drop folder. + */ +@Testcontainers(disabledWithoutDocker = true) +@EnabledIfEnvironmentVariable( + named = "RUN_NETWORK_INTEGRATION_TESTS", + matches = "true", + disabledReason = + "Spins up SFTP/FTP/SMB containers; opt-in to keep several heavy containers off the" + + " standard CI runner. Run with RUN_NETWORK_INTEGRATION_TESTS=true.") +class SftpNetworkSourceIntegrationTest { + + private static final String POLICY = "p1"; + private static final String USER = "stirling"; + private static final String PASS = "secret"; + private static final String DIR = "upload"; + private static final int SFTP_PORT = 22; + + @org.testcontainers.junit.jupiter.Container + static GenericContainer sftp = + new GenericContainer<>("atmoz/sftp:alpine") + .withExposedPorts(SFTP_PORT) + .withCommand(USER + ":" + PASS + ":::" + DIR) + .waitingFor(Wait.forListeningPort()); + + private NetworkInputSource source; + private InProcessProcessedLedger ledger; + private RecordingContext ctx; + + @BeforeEach + void setUp() throws IOException { + // Each run's container has a fresh host key, so drop pins from earlier runs: a reused + // mapped port would otherwise trip the trust-on-first-use "key changed" refusal. + Files.deleteIfExists( + Path.of(InstallationPathConfig.getConfigPath(), SftpFileClient.KNOWN_HOSTS_FILE)); + // The container resolves to loopback, so the private-host opt-in must be on. + ApplicationProperties properties = new ApplicationProperties(); + properties.getPolicies().setAllowPrivateNetworkSources(true); + RemoteFileClientFactory factory = + new RemoteFileClientFactory(new NetworkHostGuard(properties)); + NetworkConnectionResolver resolver = + new NetworkConnectionResolver( + mock(IntegrationConfigRepository.class), + mock(OwnershipService.class), + mock(UserService.class)); + source = new NetworkInputSource(resolver, factory); + ledger = new InProcessProcessedLedger(); + ctx = new RecordingContext(); + } + + @Test + void consumeListsStreamsAndDeletes() throws Exception { + put("doc.pdf", "hello sftp"); + + List work = source.resolve(spec(Map.of()), ctx); + + assertThat(work).hasSize(1); + String identity = + "sftp://" + + sftp.getHost() + + ":" + + sftp.getMappedPort(SFTP_PORT) + + "/upload/doc.pdf"; + assertThat(ctx.present).containsExactly(identity); + assertThat(read(work.get(0))).isEqualTo("hello sftp"); + // In flight: a second sweep does not re-claim it. + assertThat(source.resolve(spec(Map.of()), ctx)).isEmpty(); + + work.get(0).onComplete().accept(true); + assertThat(exists("doc.pdf")).isFalse(); + assertThat(source.resolve(spec(Map.of()), ctx)).isEmpty(); + } + + @Test + void aFailedFileStaysOnTheServer() throws Exception { + put("doc.pdf", "data"); + + source.resolve(spec(Map.of()), ctx).get(0).onComplete().accept(false); + + assertThat(exists("doc.pdf")).isTrue(); + assertThat(source.resolve(spec(Map.of()), ctx)).isEmpty(); + } + + @Test + void validateConnectsAndRejectsBadCredentials() { + source.validate(spec(Map.of())); + + Map wrong = new HashMap<>(baseOptions()); + wrong.put("password", "not-the-password"); + assertThatThrownBy(() -> source.validate(new InputSpec("sftp", wrong))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("cannot access"); + } + + @Test + void acceptsTheServersRealHostKeyFingerprint() throws Exception { + String fingerprint = + exec("ssh-keygen -lf /etc/ssh/ssh_host_ed25519_key.pub | awk '{print $2}'").trim(); + Map pinned = new HashMap<>(baseOptions()); + pinned.put("hostKeyFingerprint", fingerprint); + source.validate(new InputSpec("sftp", pinned)); + } + + @Test + void rejectsAWrongHostKeyFingerprint() { + Map wrong = new HashMap<>(baseOptions()); + wrong.put("hostKeyFingerprint", "SHA256:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); + // Refused during the handshake, before any credentials are sent. + assertThatThrownBy(() -> source.validate(new InputSpec("sftp", wrong))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("cannot access"); + } + + @Test + void refusesAChangedHostKey() throws Exception { + source.validate(spec(Map.of())); // pins the current key (trust-on-first-use) + + Path knownHosts = + Path.of(InstallationPathConfig.getConfigPath(), SftpFileClient.KNOWN_HOSTS_FILE); + String pinned = Files.readString(knownHosts); + // Swap the pinned key for a valid-but-different one so the server's real key reads as + // changed rather than unparseable. + Files.writeString(knownHosts, pinned.replaceFirst("(?m) [^ ]+$", " " + bogusEd25519Key())); + + assertThatThrownBy(() -> source.validate(spec(Map.of()))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("cannot access"); + } + + /** A structurally valid ssh-ed25519 public key blob (all-zero key material), base64. */ + private static String bogusEd25519Key() { + byte[] type = "ssh-ed25519".getBytes(StandardCharsets.US_ASCII); + ByteBuffer blob = ByteBuffer.allocate(4 + type.length + 4 + 32); + blob.putInt(type.length).put(type).putInt(32); + return java.util.Base64.getEncoder().encodeToString(blob.array()); + } + + private Map baseOptions() { + Map options = new HashMap<>(); + options.put("protocol", "sftp"); + options.put("host", sftp.getHost()); + options.put("port", String.valueOf(sftp.getMappedPort(SFTP_PORT))); + options.put("username", USER); + options.put("password", PASS); + options.put("directory", DIR); + return options; + } + + private InputSpec spec(Map extra) { + Map options = new HashMap<>(baseOptions()); + options.putAll(extra); + return new InputSpec("sftp", options); + } + + private void put(String name, String content) throws Exception { + String path = "/home/" + USER + "/" + DIR + "/" + name; + exec("printf '%s' '" + content + "' > " + path + " && chmod 644 " + path); + } + + private boolean exists(String name) throws Exception { + return exec("test -f /home/" + USER + "/" + DIR + "/" + name + " && echo yes || echo no") + .contains("yes"); + } + + private String exec(String script) throws Exception { + Container.ExecResult result = sftp.execInContainer("sh", "-c", script); + return result.getStdout() + result.getStderr(); + } + + private static String read(ResolvedInput unit) throws IOException { + try (InputStream stream = unit.inputs().primary().get(0).getInputStream()) { + return new String(stream.readAllBytes(), StandardCharsets.UTF_8); + } + } + + private class RecordingContext implements ResolveContext { + + private final List present = new ArrayList<>(); + + @Override + public boolean claim(String identity, String gate, Supplier contentHash) { + return ledger.claim(POLICY, identity, gate, contentHash); + } + + @Override + public void settle( + String identity, String finalGate, String finalContentHash, boolean success) { + ledger.settle(POLICY, identity, finalGate, finalContentHash, success); + } + + @Override + public boolean allSettledDone(String identity) { + return ledger.allSettledDone(identity); + } + + @Override + public void reportPresent(Collection identities) { + present.addAll(identities); + } + } +} diff --git a/app/proprietary/src/test/java/stirling/software/proprietary/policy/network/SmbNetworkSourceIntegrationTest.java b/app/proprietary/src/test/java/stirling/software/proprietary/policy/network/SmbNetworkSourceIntegrationTest.java new file mode 100644 index 0000000000..9d07100bc4 --- /dev/null +++ b/app/proprietary/src/test/java/stirling/software/proprietary/policy/network/SmbNetworkSourceIntegrationTest.java @@ -0,0 +1,194 @@ +package stirling.software.proprietary.policy.network; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.junit.jupiter.Testcontainers; + +import stirling.software.common.model.ApplicationProperties; +import stirling.software.proprietary.access.service.OwnershipService; +import stirling.software.proprietary.integration.repository.IntegrationConfigRepository; +import stirling.software.proprietary.policy.input.ResolveContext; +import stirling.software.proprietary.policy.input.ResolvedInput; +import stirling.software.proprietary.policy.ledger.InProcessProcessedLedger; +import stirling.software.proprietary.policy.model.InputSpec; +import stirling.software.proprietary.security.service.UserService; + +/** + * End-to-end {@link NetworkInputSource} test over SMB against a real Samba server, through the + * production {@link SmbFileClient}: listing a share, claiming, streaming the actual bytes, consume + * delete, and save-time validation. The share is served world-writable so the SMB user can remove a + * consumed file. + */ +@Testcontainers(disabledWithoutDocker = true) +@EnabledIfEnvironmentVariable( + named = "RUN_NETWORK_INTEGRATION_TESTS", + matches = "true", + disabledReason = + "Spins up SFTP/FTP/SMB containers; opt-in to keep several heavy containers off the" + + " standard CI runner. Run with RUN_NETWORK_INTEGRATION_TESTS=true.") +class SmbNetworkSourceIntegrationTest { + + private static final String POLICY = "p1"; + private static final String USER = "stirling"; + private static final String PASS = "secret"; + private static final String SHARE = "documents"; + private static final int SMB_PORT = 445; + + @org.testcontainers.junit.jupiter.Container + static GenericContainer samba = + new GenericContainer<>("dperson/samba") + .withExposedPorts(SMB_PORT) + .withCommand( + "-p", + "-u", + USER + ";" + PASS, + "-s", + SHARE + ";/share;yes;no;no;" + USER) + .waitingFor(Wait.forListeningPort()); + + private NetworkInputSource source; + private InProcessProcessedLedger ledger; + private RecordingContext ctx; + + @BeforeEach + void setUp() throws Exception { + // The share dir must be writable by the SMB user so a consumed file can be deleted. + samba.execInContainer("sh", "-c", "mkdir -p /share && chmod -R 0777 /share"); + ApplicationProperties properties = new ApplicationProperties(); + properties.getPolicies().setAllowPrivateNetworkSources(true); + RemoteFileClientFactory factory = + new RemoteFileClientFactory(new NetworkHostGuard(properties)); + NetworkConnectionResolver resolver = + new NetworkConnectionResolver( + mock(IntegrationConfigRepository.class), + mock(OwnershipService.class), + mock(UserService.class)); + source = new NetworkInputSource(resolver, factory); + ledger = new InProcessProcessedLedger(); + ctx = new RecordingContext(); + } + + @Test + void consumeListsStreamsAndDeletes() throws Exception { + put("doc.pdf", "hello smb"); + + List work = source.resolve(spec(Map.of()), ctx); + + assertThat(work).hasSize(1); + String identity = + "smb://" + + samba.getHost() + + ":" + + samba.getMappedPort(SMB_PORT) + + "/documents/doc.pdf"; + assertThat(ctx.present).containsExactly(identity); + assertThat(read(work.get(0))).isEqualTo("hello smb"); + assertThat(source.resolve(spec(Map.of()), ctx)).isEmpty(); + + work.get(0).onComplete().accept(true); + assertThat(exists("doc.pdf")).isFalse(); + assertThat(source.resolve(spec(Map.of()), ctx)).isEmpty(); + } + + @Test + void aFailedFileStaysOnTheShare() throws Exception { + put("doc.pdf", "data"); + + source.resolve(spec(Map.of()), ctx).get(0).onComplete().accept(false); + + assertThat(exists("doc.pdf")).isTrue(); + assertThat(source.resolve(spec(Map.of()), ctx)).isEmpty(); + } + + @Test + void validateConnectsAndRejectsBadCredentials() { + source.validate(spec(Map.of())); + + Map wrong = new HashMap<>(baseOptions()); + wrong.put("password", "not-the-password"); + assertThatThrownBy(() -> source.validate(new InputSpec("network", wrong))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("cannot access"); + } + + private Map baseOptions() { + Map options = new HashMap<>(); + options.put("protocol", "smb"); + options.put("host", samba.getHost()); + options.put("port", String.valueOf(samba.getMappedPort(SMB_PORT))); + options.put("username", USER); + options.put("password", PASS); + options.put("share", SHARE); + return options; + } + + private InputSpec spec(Map extra) { + Map options = new HashMap<>(baseOptions()); + options.putAll(extra); + return new InputSpec("network", options); + } + + private void put(String name, String content) throws Exception { + String path = "/share/" + name; + exec("printf '%s' '" + content + "' > " + path + " && chmod 0666 " + path); + } + + private boolean exists(String name) throws Exception { + return exec("test -f /share/" + name + " && echo yes || echo no").contains("yes"); + } + + private String exec(String script) throws Exception { + Container.ExecResult result = samba.execInContainer("sh", "-c", script); + return result.getStdout() + result.getStderr(); + } + + private static String read(ResolvedInput unit) throws IOException { + try (InputStream stream = unit.inputs().primary().get(0).getInputStream()) { + return new String(stream.readAllBytes(), StandardCharsets.UTF_8); + } + } + + private class RecordingContext implements ResolveContext { + + private final List present = new ArrayList<>(); + + @Override + public boolean claim(String identity, String gate, Supplier contentHash) { + return ledger.claim(POLICY, identity, gate, contentHash); + } + + @Override + public void settle( + String identity, String finalGate, String finalContentHash, boolean success) { + ledger.settle(POLICY, identity, finalGate, finalContentHash, success); + } + + @Override + public boolean allSettledDone(String identity) { + return ledger.allSettledDone(identity); + } + + @Override + public void reportPresent(Collection identities) { + present.addAll(identities); + } + } +} diff --git a/build.gradle b/build.gradle index 28e20c9b15..3bcae55ea7 100644 --- a/build.gradle +++ b/build.gradle @@ -44,6 +44,9 @@ ext { jpdfiumVersion = "1.0.2" jwtVersion = "0.13.0" awsSdkVersion = "2.44.12" + jschVersion = "0.2.23" + commonsNetVersion = "3.11.1" + smbjVersion = "0.14.0" tinkVersion = "1.23.0" testcontainersMinioVersion = "1.21.4" // junit-platform-launcher version managed by Spring Boot BOM diff --git a/frontend/editor/public/locales/en-US/translation.toml b/frontend/editor/public/locales/en-US/translation.toml index 25a2e4bafa..77e41e2ffd 100644 --- a/frontend/editor/public/locales/en-US/translation.toml +++ b/frontend/editor/public/locales/en-US/translation.toml @@ -6701,9 +6701,15 @@ label = "Server URL" [portal.connections.commonFields.email] label = "Email" +[portal.connections.commonFields.host] +label = "Host" + [portal.connections.commonFields.password] label = "Password" +[portal.connections.commonFields.port] +label = "Port" + [portal.connections.commonFields.username] label = "Username" @@ -6836,6 +6842,27 @@ baseUrlPlaceholder = "https://your-cluster:9200" description = "Index an audit event in Elasticsearch each time a policy handles a document." label = "Elasticsearch" +[portal.connections.types.ftp] +description = "Poll an FTP or FTPS server for new documents." +hostPlaceholder = "ftp.example.com" +label = "FTP / FTPS" + +[portal.connections.types.ftp.fields.passive] +label = "Connection mode" + +[portal.connections.types.ftp.fields.passive.options] +active = "Active" +passive = "Passive (recommended)" + +[portal.connections.types.ftp.fields.security] +helperText = "Plain FTP sends credentials and files unencrypted; choose FTPS if the server supports it." +label = "Encryption" + +[portal.connections.types.ftp.fields.security.options] +explicit = "FTPS (explicit / AUTH TLS)" +implicit = "FTPS (implicit)" +none = "None (plain FTP)" + [portal.connections.types.googlechat] baseUrlPlaceholder = "https://chat.googleapis.com/v1/spaces/..." description = "Post a message to a Google Chat space when a policy runs." @@ -6903,10 +6930,43 @@ label = "Secret access key" description = "Email a document or a notification when a policy runs." label = "SendGrid" +[portal.connections.types.sftp] +description = "Poll an SFTP (SSH) server for new documents." +hostPlaceholder = "sftp.example.com" +label = "SFTP" + +[portal.connections.types.sftp.fields.hostKeyFingerprint] +helperText = "Optional. The server's SHA-256 host key fingerprint (from `ssh-keyscan host | ssh-keygen -lf -`). When set, only a server presenting this key is accepted; when blank, the key seen on first connect is pinned and later changes are refused." +label = "Host key fingerprint" +placeholder = "SHA256:..." + +[portal.connections.types.sftp.fields.passphrase] +label = "Key passphrase" + +[portal.connections.types.sftp.fields.password] +helperText = "Leave blank if you authenticate with a private key instead." + +[portal.connections.types.sftp.fields.privateKey] +helperText = "Optional. Paste an OpenSSH private key to authenticate with a key instead of a password." +label = "Private key" + [portal.connections.types.slack] description = "Post a message to Slack when a policy processes a document." label = "Slack" +[portal.connections.types.smb] +description = "Watch an SMB / CIFS network share (Windows or Samba)." +hostPlaceholder = "fileserver.example.com" +label = "SMB / Network share" + +[portal.connections.types.smb.fields.domain] +helperText = "Optional. The Windows domain or workgroup, if your server uses one." +label = "Domain" + +[portal.connections.types.smb.fields.share] +label = "Share" +placeholder = "documents" + [portal.connections.types.splunk] baseUrlPlaceholder = "https://your-splunk:8088" description = "Send an audit event to Splunk each time a policy handles a document." @@ -8711,6 +8771,30 @@ inUse = "In use" total = "Connections" unused = "Unused" +[portal.sources.networkFields.connection] +helperText = "The stored connection with the server address and credentials. Reused by every source that references it." +label = "Connection" + +[portal.sources.networkFields.directory] +helperText = "Folder on the server to poll, relative to the login home or share root. Leave blank for the root." +label = "Folder" +placeholder = "incoming/" + +[portal.sources.networkFields.mode] +helperText = "Consume removes each file from the server once every policy has processed it." +label = "Read mode" + +[portal.sources.networkFields.mode.options] +consume = "Consume: process each file once" +snapshot = "Snapshot: re-read the folder every run" + +[portal.sources.networkFields.recursive] +label = "Folder depth" + +[portal.sources.networkFields.recursive.options] +all = "Include subfolders" +top = "Top level only" + [portal.sources.status] active = "Active" disabled = "Disabled" @@ -8770,6 +8854,10 @@ label = "Folder depth" all = "Include subfolders" top = "Top level only" +[portal.sources.types.ftp] +description = "Poll an FTP or FTPS server folder for new documents." +label = "FTP" + [portal.sources.types.googledrive] description = "Pull documents from shared Google Drive folders." label = "Google Drive" diff --git a/frontend/editor/src/portal/api/integrations.ts b/frontend/editor/src/portal/api/integrations.ts index 01eebe96ab..4ced16ca79 100644 --- a/frontend/editor/src/portal/api/integrations.ts +++ b/frontend/editor/src/portal/api/integrations.ts @@ -6,7 +6,13 @@ */ import { apiClient } from "@portal/api/http"; -export type IntegrationType = "S3" | "MCP" | "API" | "PURVIEW" | "CONSIGNO"; +export type IntegrationType = + | "S3" + | "NETWORK" + | "MCP" + | "API" + | "PURVIEW" + | "CONSIGNO"; export type OwnerScope = "USER" | "TEAM" | "SERVER"; /** Mirrors the backend IntegrationConfigResponse; `config` values are masked. */ diff --git a/frontend/editor/src/portal/components/BrandMarks.tsx b/frontend/editor/src/portal/components/BrandMarks.tsx index 04f1d79e83..7c381650d3 100644 --- a/frontend/editor/src/portal/components/BrandMarks.tsx +++ b/frontend/editor/src/portal/components/BrandMarks.tsx @@ -383,6 +383,14 @@ const NEUTRAL: Record = { ), + ftp: ( + <> + + + + + + ), email: ( <> diff --git a/frontend/editor/src/portal/components/sources/ConnectionForm.tsx b/frontend/editor/src/portal/components/sources/ConnectionForm.tsx index 79d6883bdb..f975e5cc9d 100644 --- a/frontend/editor/src/portal/components/sources/ConnectionForm.tsx +++ b/frontend/editor/src/portal/components/sources/ConnectionForm.tsx @@ -26,8 +26,24 @@ export function ConnectionForm({ onChange, }: ConnectionFormProps) { const { t } = useTranslation(); - const set = (key: string, value: string) => - onChange({ ...values, [key]: value }); + const set = (key: string, value: string) => { + const next = { ...values, [key]: value }; + const sync = type.fields.find((f) => f.key === key)?.syncsDefault; + if (sync) { + const target = next[sync.targetKey] ?? ""; + const targetDefault = + type.fields.find((f) => f.key === sync.targetKey)?.defaultValue ?? ""; + const untouched = + target === "" || + target === targetDefault || + Object.values(sync.map).includes(target); + const mapped = sync.map[value]; + if (untouched && mapped !== undefined) { + next[sync.targetKey] = mapped; + } + } + onChange(next); + }; return (

diff --git a/frontend/editor/src/portal/components/sources/SourceModal.tsx b/frontend/editor/src/portal/components/sources/SourceModal.tsx index 626c12b33a..0235b22a32 100644 --- a/frontend/editor/src/portal/components/sources/SourceModal.tsx +++ b/frontend/editor/src/portal/components/sources/SourceModal.tsx @@ -32,12 +32,14 @@ import { } from "@portal/components/sources/sourceTypes"; import { BrandMark } from "@portal/components/BrandMarks"; import { S3ConnectionPicker } from "@portal/components/sources/S3ConnectionPicker"; +import { ConnectionPicker } from "@portal/components/sources/ConnectionPicker"; import { ConnectionForm } from "@portal/components/sources/ConnectionForm"; import { CREATABLE_CONNECTION_TYPES, buildConnectionConfig, connectionFormValid, emptyConnectionValues, + type CreatableConnectionType, } from "@portal/components/sources/connectionTypes"; import { createIntegration } from "@portal/api/integrations"; import "@portal/components/sources/SourceModal.css"; @@ -76,6 +78,14 @@ const S3_CONNECTION_TYPE = CREATABLE_CONNECTION_TYPES.find( (entry) => entry.id === "s3", )!; +/** A connection-catalogue entry by id, for the inline create form a source field opens. */ +function connectionTypeById(id: string): CreatableConnectionType { + return ( + CREATABLE_CONNECTION_TYPES.find((entry) => entry.id === id) ?? + S3_CONNECTION_TYPE + ); +} + interface SourceModalProps { open: boolean; /** When set, edit this source; otherwise create a new one. */ @@ -129,7 +139,10 @@ export function SourceModal({ webhookId: string; secret: string; } | null>(null); - // In-place connection create (swaps the stage; never stacks a second modal). + // In-place connection create (swaps the stage; never stacks a second modal). The type is set + // per field, so a source that wants an SFTP connection opens the SFTP form, not always S3. + const [connType, setConnType] = + useState(S3_CONNECTION_TYPE); const [connValues, setConnValues] = useState>(() => emptyConnectionValues(S3_CONNECTION_TYPE), ); @@ -192,24 +205,27 @@ export function SourceModal({ onClose(); } - function openConnectionStage(fieldKey: string) { - setConnValues(emptyConnectionValues(S3_CONNECTION_TYPE)); + function openConnectionStage( + fieldKey: string, + type: CreatableConnectionType, + ) { + setConnType(type); + setConnValues(emptyConnectionValues(type)); setConnField(fieldKey); setError(null); setStage("connection"); } async function saveConnection() { - if (connSaving || !connectionFormValid(S3_CONNECTION_TYPE, connValues)) - return; + if (connSaving || !connectionFormValid(connType, connValues)) return; setConnSaving(true); setError(null); try { const created = await createIntegration({ - integrationType: S3_CONNECTION_TYPE.integrationType, + integrationType: connType.integrationType, name: connValues.name.trim(), scope: "TEAM", - config: buildConnectionConfig(S3_CONNECTION_TYPE, connValues), + config: buildConnectionConfig(connType, connValues), }); // Back to the source form with the fresh connection selected; the picker // remounts and refetches, so the new name is in its list. @@ -277,7 +293,7 @@ export function SourceModal({ ? t("portal.sources.builder.createTitle") : stage === "connection" ? t("portal.connections.createTitleFor", { - name: t(S3_CONNECTION_TYPE.labelKey), + name: t(connType.labelKey), }) : stage === "reveal" ? t("portal.sources.types.webhook.reveal.title") @@ -346,7 +362,7 @@ export function SourceModal({