diff --git a/.semaphore/post_install.yml b/.semaphore/post_install.yml index 6c7a83a4..66c39c96 100644 --- a/.semaphore/post_install.yml +++ b/.semaphore/post_install.yml @@ -36,7 +36,7 @@ blocks: - env_var: DOCKER_IMAGE values: ["debian:bullseye", "debian:bookworm", "ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"] - env_var: NODE_VERSION - values: ["18", "20", "21", "22", "23"] + values: ["18", "20", "21", "22", "23", "24"] commands: - docker run -v "$(pwd):/v" "$DOCKER_IMAGE" /v/ci/post-install/install-debian.sh "$NODE_VERSION" "$LIBRARY_VERSION" @@ -55,7 +55,7 @@ blocks: - env_var: DOCKER_IMAGE values: ["debian:bullseye", "debian:bookworm", "ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"] - env_var: NODE_VERSION - values: ["18", "20", "21", "22", "23"] + values: ["18", "20", "21", "22", "23", "24"] commands: - docker run -v "$(pwd):/v" "$DOCKER_IMAGE" /v/ci/post-install/install-debian.sh "$NODE_VERSION" "$LIBRARY_VERSION" @@ -72,7 +72,7 @@ blocks: - name: "Install" matrix: - env_var: NODE_VERSION - values: ["18", "20", "21", "22", "23"] + values: ["18", "20", "21", "22", "23", "24"] commands: - docker run -v "$(pwd):/v" "node:${NODE_VERSION}-alpine" /v/ci/post-install/install-alpine.sh "$NODE_VERSION" "$LIBRARY_VERSION" @@ -89,7 +89,7 @@ blocks: - name: "Install" matrix: - env_var: NODE_VERSION - values: ["18", "20", "21", "22", "23"] + values: ["18", "20", "21", "22", "23", "24"] commands: - docker run -v "$(pwd):/v" "node:${NODE_VERSION}-alpine" /v/ci/post-install/install-alpine.sh "$NODE_VERSION" "$LIBRARY_VERSION" @@ -108,7 +108,7 @@ blocks: - env_var: DOCKER_IMAGE values: ["quay.io/centos/centos:stream9", "almalinux:9", "rockylinux:9"] - env_var: NODE_VERSION - values: ["18", "20", "21", "22", "23"] + values: ["18", "20", "21", "22", "23", "24"] commands: - docker run -v "$(pwd):/v" "$DOCKER_IMAGE" /v/ci/post-install/install-rhel.sh "$NODE_VERSION" "$LIBRARY_VERSION" @@ -127,7 +127,7 @@ blocks: - env_var: DOCKER_IMAGE values: ["quay.io/centos/centos:stream9", "almalinux:9", "rockylinux:9"] - env_var: NODE_VERSION - values: ["18", "20", "21", "22", "23"] + values: ["18", "20", "21", "22", "23", "24"] commands: - docker run -v "$(pwd):/v" "$DOCKER_IMAGE" /v/ci/post-install/install-rhel.sh "$NODE_VERSION" "$LIBRARY_VERSION" @@ -141,7 +141,7 @@ blocks: - name: "Install" matrix: - env_var: NODE_VERSION - values: ["18", "20", "21", "22", "23"] + values: ["18", "20", "21", "22", "23", "24"] commands: - sem-version node $NODE_VERSION - bash ci/post-install/install-mac.sh "$NODE_VERSION" "$LIBRARY_VERSION" @@ -156,7 +156,7 @@ blocks: - name: "Install" matrix: - env_var: NODE_VERSION - values: ["18", "20", "21", "22", "23"] + values: ["18", "20", "21", "22", "23", "24"] commands: - sem-version node $NODE_VERSION - bash ci/post-install/install-mac.sh "$NODE_VERSION" "$LIBRARY_VERSION" @@ -187,7 +187,7 @@ blocks: - name: "Install" matrix: - env_var: NODE_VERSION - values: ["18.19.0", "20.11.0", "21.6.1", "22.2.0", "23.2.0"] + values: ["18.19.0", "20.11.0", "21.6.1", "22.2.0", "23.2.0", "24.3.0"] commands: - Invoke-WebRequest "https://nodejs.org/download/release/v${env:NODE_VERSION}/node-v${env:NODE_VERSION}-x64.msi" -OutFile node.msi - msiexec /qn /l* node-log.txt /i node.msi diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index d7b1125c..1bbcab62 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -28,7 +28,7 @@ global_job_config: - git submodule update --init --recursive - cd deps/librdkafka - git fetch origin - - git checkout v2.10.1 + - git checkout v2.11.0-RC3 - cd ../../ - cache clear @@ -209,6 +209,7 @@ blocks: - 21,120 - 22,127 - 23,131 + - 24,137 commands: - export NODE_VERSION=$(echo $NODE_VERSION_ABI | cut -d, -f1) - export NODE_ABI=$(echo $NODE_VERSION_ABI | cut -d, -f2) @@ -245,6 +246,7 @@ blocks: - 21,120 - 22,127 - 23,131 + - 24,137 commands: - export NODE_VERSION=$(echo $NODE_VERSION_ABI | cut -d, -f1) - export NODE_ABI=$(echo $NODE_VERSION_ABI | cut -d, -f2) @@ -281,6 +283,7 @@ blocks: - 21,120 - 22,127 - 23,131 + - 24,137 commands: - export NODE_VERSION=$(echo $NODE_VERSION_ABI | cut -d, -f1) - export NODE_ABI=$(echo $NODE_VERSION_ABI | cut -d, -f2) @@ -317,6 +320,7 @@ blocks: - 21,120 - 22,127 - 23,131 + - 24,137 commands: - export NODE_VERSION=$(echo $NODE_VERSION_ABI | cut -d, -f1) - export NODE_ABI=$(echo $NODE_VERSION_ABI | cut -d, -f2) @@ -350,6 +354,7 @@ blocks: - 21,120 - 22,127 - 23,131 + - 24,137 commands: - export NODE_VERSION=$(echo $NODE_VERSION_ABI | cut -d, -f1) - export NODE_ABI=$(echo $NODE_VERSION_ABI | cut -d, -f2) @@ -385,6 +390,7 @@ blocks: - 21,120 - 22,127 - 23,131 + - 24,137 commands: - export NODE_VERSION=$(echo $NODE_VERSION_ABI | cut -d, -f1) - export NODE_ABI=$(echo $NODE_VERSION_ABI | cut -d, -f2) @@ -439,6 +445,7 @@ blocks: - 21.6.1,120 - 22.2.0,127 - 23.2.0,131 + - 24.3.0,137 commands: - $env:NODE_VERSION = $env:NODE_VERSION_ABI.Split(',')[0] - $env:NODE_ABI = $env:NODE_VERSION_ABI.Split(',')[1] @@ -449,6 +456,7 @@ blocks: - pip install setuptools - $env:ARTIFACT_KEY = "confluent-kafka-javascript-${env:SEMAPHORE_GIT_TAG_NAME}-node-v${env:NODE_ABI}-${env:PLATFORM}-${env:LIBC}-${env:ARCHITECTURE}.tar.gz" - echo "$env:ARTIFACT_KEY" + - npm config delete registry # we have no requirement for internal packages. - npm ci # node-pre-gyp will fallback to build here, because new tag implies no release yet. - npx node-pre-gyp package - ls "build/stage/${env:SEMAPHORE_GIT_TAG_NAME}/${env:ARTIFACT_KEY}" diff --git a/CHANGELOG.md b/CHANGELOG.md index beadc7a0..44ac7cc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,8 +5,10 @@ v1.4.0 is a feature release. It is supported for all usage. ## Enhancements 1. References librdkafka v2.11.0. Refer to the [librdkafka v2.11.0 release notes](https://github.com/confluentinc/librdkafka/releases/tag/v2.11.0) for more information. -2. [KIP-848] `describeGroups()` now supports KIP-848 introduced `consumer` groups. Two new fields for consumer group type and target assignment have also been added. Type defines whether this group is a `classic` or `consumer` group. Target assignment is only valid for the `consumer` protocol and its defaults to being undefined (#329) -3. [KIP-848] Admin API for listing consumer groups now has an optional filter to return only groups of given types (#328) +2. [KIP-848] `describeGroups()` now supports KIP-848 introduced `consumer` groups. Two new fields for consumer group type and target assignment have also been added. Type defines whether this group is a `classic` or `consumer` group. Target assignment is only valid for the `consumer` protocol and it defaults to being undefined (#329). +3. [KIP-848] Admin API for listing consumer groups now has an optional filter to return only groups of given types (#328). +4. Add support for Node v24 pre-built binaries (@weyert, #307, #337). + # confluent-kafka-javascript v1.3.2 diff --git a/README.md b/README.md index c4e6a7d4..4cd2fe1d 100644 --- a/README.md +++ b/README.md @@ -33,15 +33,15 @@ The following configurations are supported: | Distribution | Supported Node Versions | | ----------------------------------------- | ----------------------- | -| Debian Bullseye/Ubuntu 20.04 | 18, 20, 21, 22, 23 | -| Debian Bookworm/Ubuntu 22.04 | 18, 20, 21, 22, 23 | -| Alpine Linux 3.20+ | 18, 20, 21, 22, 23 | -| AlmaLinux 9/Rocky Linux 9/CentOS Stream 9 | 18, 20, 21, 22, 23 | +| Debian Bullseye/Ubuntu 20.04 | 18, 20, 21, 22, 23, 24 | +| Debian Bookworm/Ubuntu 22.04 | 18, 20, 21, 22, 23, 24 | +| Alpine Linux 3.20+ | 18, 20, 21, 22, 23, 24 | +| AlmaLinux 9/Rocky Linux 9/CentOS Stream 9 | 18, 20, 21, 22, 23, 24 | Other distributions will probably work given a modern version of gcc/glibc, but we don't test the pre-built binaries with them. -* macOS - arm64/m1. macOS (Intel) is supported on a best-effort basis. Node versions 18, 20, 21, 22, and 23 are supported. -* Windows - x64. Node versions 18, 20, 21, 22, and 23 are supported. +* macOS - arm64/m1. macOS (Intel) is supported on a best-effort basis. Node versions 18, 20, 21, 22, 23 and 24 are supported. +* Windows - x64. Node versions 18, 20, 21, 22, 23 and 24 are supported. > [!WARNING] > Pre-built binary support will be dropped after the EOL of the node version or the OS. @@ -115,6 +115,7 @@ For guidelines on contributing please see [CONTRIBUTING.md](CONTRIBUTING.md) | 1.3.0 | 2.10.0 | | 1.3.1 | 2.10.0 | | 1.3.2 | 2.10.1 | +| 1.4.0 | 2.11.0 | This mapping is applicable if you're using a pre-built binary. Otherwise, you can check the librdkafka version with the following command: diff --git a/deps/librdkafka b/deps/librdkafka index 2af67492..ec4cf936 160000 --- a/deps/librdkafka +++ b/deps/librdkafka @@ -1 +1 @@ -Subproject commit 2af6749287b3695edf2421631ebf08d6bcb176e3 +Subproject commit ec4cf9360592bc0b789d2fb2f6601eec1c128a2e diff --git a/deps/windows-install.py b/deps/windows-install.py index 635141b8..685d80e9 100644 --- a/deps/windows-install.py +++ b/deps/windows-install.py @@ -16,10 +16,13 @@ depsIncludeDir = '../deps/include' buildReleaseDir = 'Release' -# alternative: 'https://api.nuget.org/v3-flatcontainer/librdkafka.redist/{}/librdkafka.redist.{}.nupkg'.format(librdkafkaVersion, librdkafkaVersion) +# Use publicly documented API to download librdkafka NuGet package. +# https://api.nuget.org/v3-flatcontainer/{package}/{version}/{package}.{version}.nupkg +# See https://learn.microsoft.com/en-us/nuget/api/package-base-address-resource#download-package-content-nupkg env_dist = os.environ -downloadBaseUrl = env_dist['NODE_RDKAFKA_NUGET_BASE_URL'] if 'NODE_RDKAFKA_NUGET_BASE_URL' in env_dist else 'https://globalcdn.nuget.org/packages/' -librdkafkaNugetUrl = downloadBaseUrl + 'librdkafka.redist.{}.nupkg'.format(librdkafkaVersion) +downloadBaseUrl = env_dist['NODE_RDKAFKA_NUGET_BASE_URL'] if 'NODE_RDKAFKA_NUGET_BASE_URL' in env_dist else 'https://api.nuget.org/v3-flatcontainer/librdkafka.redist/{version}/' +librdkafkaNugetUrl = downloadBaseUrl + 'librdkafka.redist.{version}.nupkg' +librdkafkaNugetUrl = librdkafkaNugetUrl.format(version=librdkafkaVersion) print('download librdkafka form ' + librdkafkaNugetUrl) outputDir = 'librdkafka.redist' outputFile = outputDir + '.zip' diff --git a/lib/error.js b/lib/error.js index eb0da33b..e569971f 100644 --- a/lib/error.js +++ b/lib/error.js @@ -28,7 +28,7 @@ LibrdKafkaError.wrap = errorWrap; * @constant * @memberof RdKafka */ -// ====== Generated from librdkafka 2.10.1 file src-cpp/rdkafkacpp.h ====== +// ====== Generated from librdkafka v2.11.0-RC3 file src-cpp/rdkafkacpp.h ====== LibrdKafkaError.codes = { /* Internal errors to rdkafka: */ diff --git a/lib/util.js b/lib/util.js index ed72e6c4..d7e0134d 100644 --- a/lib/util.js +++ b/lib/util.js @@ -52,4 +52,4 @@ util.dictToStringList = function (mapOrObject) { return list; }; -util.bindingVersion = '1.3.2'; +util.bindingVersion = '1.4.0-rc0-dev'; diff --git a/package-lock.json b/package-lock.json index ddd9110d..9bb56308 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@confluentinc/kafka-javascript", - "version": "1.3.2", + "version": "1.4.0-rc0-dev", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@confluentinc/kafka-javascript", - "version": "1.3.2", + "version": "1.4.0-rc0-dev", "hasInstallScript": true, "license": "MIT", "workspaces": [ @@ -10952,7 +10952,7 @@ }, "schemaregistry": { "name": "@confluentinc/schemaregistry", - "version": "1.3.1", + "version": "1.4.0-rc0-dev", "license": "MIT", "dependencies": { "@aws-sdk/client-kms": "^3.637.0", diff --git a/package.json b/package.json index 48458207..ae45c298 100644 --- a/package.json +++ b/package.json @@ -1,9 +1,9 @@ { "name": "@confluentinc/kafka-javascript", - "version": "1.3.2", + "version": "1.4.0-rc0-dev", "description": "Node.js bindings for librdkafka", - "librdkafka": "2.10.1", - "librdkafka_win": "2.10.1", + "librdkafka": "2.11.0-RC3", + "librdkafka_win": "2.11.0-RC3", "main": "lib/index.js", "types": "types/index.d.ts", "scripts": { @@ -65,4 +65,4 @@ "schemaregistry", "schemaregistry-examples" ] -} \ No newline at end of file +} diff --git a/schemaregistry/docker-compose.schemaregistry.yml b/schemaregistry/docker-compose.schemaregistry.yml index 98bcca80..3bb2342a 100644 --- a/schemaregistry/docker-compose.schemaregistry.yml +++ b/schemaregistry/docker-compose.schemaregistry.yml @@ -5,7 +5,7 @@ services: environment: ZOOKEEPER_CLIENT_PORT: 2181 kafka: - image: confluentinc/cp-kafka + image: confluentinc/cp-kafka:7.9.2 restart: always depends_on: - zookeeper diff --git a/schemaregistry/package.json b/schemaregistry/package.json index cef2d8ac..f7ac540f 100644 --- a/schemaregistry/package.json +++ b/schemaregistry/package.json @@ -1,6 +1,6 @@ { "name": "@confluentinc/schemaregistry", - "version": "1.3.1", + "version": "1.4.0-rc0-dev", "description": "Node.js client for Confluent Schema Registry", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/types/config.d.ts b/types/config.d.ts index fcd62b76..c7326914 100644 --- a/types/config.d.ts +++ b/types/config.d.ts @@ -1,4 +1,4 @@ -// ====== Generated from librdkafka 2.10.1 file CONFIGURATION.md ====== +// ====== Generated from librdkafka v2.11.0-RC3 file CONFIGURATION.md ====== // Code that generated this is a derivative work of the code from Nam Nguyen // https://gist.github.com/ntgn81/066c2c8ec5b4238f85d1e9168a04e3fb @@ -63,12 +63,19 @@ export interface GlobalConfig { "max.in.flight"?: number; /** - * Controls how the client recovers when none of the brokers known to it is available. If set to `none`, the client fails with a fatal error. If set to `rebootstrap`, the client repeats the bootstrap process using `bootstrap.servers` and brokers added through `rd_kafka_brokers_add()`. Rebootstrapping is useful when a client communicates with brokers so infrequently that the set of brokers may change entirely before the client refreshes metadata. Metadata recovery is triggered when all last-known brokers appear unavailable simultaneously. + * Controls how the client recovers when none of the brokers known to it is available. If set to `none`, the client doesn't re-bootstrap. If set to `rebootstrap`, the client repeats the bootstrap process using `bootstrap.servers` and brokers added through `rd_kafka_brokers_add()`. Rebootstrapping is useful when a client communicates with brokers so infrequently that the set of brokers may change entirely before the client refreshes metadata. Metadata recovery is triggered when all last-known brokers appear unavailable simultaneously or the client cannot refresh metadata within `metadata.recovery.rebootstrap.trigger.ms` or it's requested in a metadata response. * * @default rebootstrap */ "metadata.recovery.strategy"?: 'none' | 'rebootstrap'; + /** + * If a client configured to rebootstrap using `metadata.recovery.strategy=rebootstrap` is unable to obtain metadata from any of the brokers for this interval, client repeats the bootstrap process using `bootstrap.servers` configuration and brokers added through `rd_kafka_brokers_add()`. + * + * @default 300000 + */ + "metadata.recovery.rebootstrap.trigger.ms"?: number; + /** * Period of time in milliseconds at which topic and broker metadata is refreshed in order to proactively discover any new brokers, topics, partitions or partition leader changes. Use -1 to disable the intervalled refresh (not recommended). If there are no locally referenced topics (no topic objects created, no messages produced, no subscription or no assignment) then only the broker list will be refreshed every interval but no more often than every 10s. * @@ -432,6 +439,16 @@ export interface GlobalConfig { */ "ssl.ca.location"?: string; + /** + * File or directory path to CA certificate(s) for verifying HTTPS endpoints, like `sasl.oauthbearer.token.endpoint.url` used for OAUTHBEARER/OIDC authentication. Mutually exclusive with `https.ca.pem`. Defaults: On Windows the system's CA certificates are automatically looked up in the Windows Root certificate store. On Mac OSX this configuration defaults to `probe`. It is recommended to install openssl using Homebrew, to provide CA certificates. On Linux install the distribution's ca-certificates package. If OpenSSL is statically linked or `https.ca.location` is set to `probe` a list of standard paths will be probed and the first one found will be used as the default CA certificate location path. If OpenSSL is dynamically linked the OpenSSL library's default path will be used (see `OPENSSLDIR` in `openssl version -a`). + */ + "https.ca.location"?: string; + + /** + * CA certificate string (PEM format) for verifying HTTPS endpoints. Mutually exclusive with `https.ca.location`. Optional: see `https.ca.location`. + */ + "https.ca.pem"?: string; + /** * CA certificate string (PEM format) for verifying the broker's key. */ @@ -591,6 +608,16 @@ export interface GlobalConfig { */ "sasl.oauthbearer.client.id"?: string; + /** + * Alias for `sasl.oauthbearer.client.id`: Public identifier for the application. Must be unique across all clients that the authorization server handles. Only used when `sasl.oauthbearer.method` is set to "oidc". + */ + "sasl.oauthbearer.client.credentials.client.id"?: string; + + /** + * Alias for `sasl.oauthbearer.client.secret`: Client secret only known to the application and the authorization server. This should be a sufficiently random string that is not guessable. Only used when `sasl.oauthbearer.method` is set to "oidc". + */ + "sasl.oauthbearer.client.credentials.client.secret"?: string; + /** * Client secret only known to the application and the authorization server. This should be a sufficiently random string that is not guessable. Only used when `sasl.oauthbearer.method` is set to "oidc". */ @@ -611,6 +638,81 @@ export interface GlobalConfig { */ "sasl.oauthbearer.token.endpoint.url"?: string; + /** + * OAuth grant type to use when communicating with the identity provider. + * + * @default client_credentials + */ + "sasl.oauthbearer.grant.type"?: 'client_credentials' | 'urn:ietf:params:oauth:grant-type:jwt-bearer'; + + /** + * Algorithm the client should use to sign the assertion sent to the identity provider and in the OAuth alg header in the JWT assertion. + * + * @default RS256 + */ + "sasl.oauthbearer.assertion.algorithm"?: 'RS256' | 'ES256'; + + /** + * Path to client's private key (PEM) used for authentication when using the JWT assertion. + */ + "sasl.oauthbearer.assertion.private.key.file"?: string; + + /** + * Private key passphrase for `sasl.oauthbearer.assertion.private.key.file` or `sasl.oauthbearer.assertion.private.key.pem`. + */ + "sasl.oauthbearer.assertion.private.key.passphrase"?: string; + + /** + * Client's private key (PEM) used for authentication when using the JWT assertion. + */ + "sasl.oauthbearer.assertion.private.key.pem"?: string; + + /** + * Path to the assertion file. Only used when `sasl.oauthbearer.method` is set to "oidc" and JWT assertion is needed. + */ + "sasl.oauthbearer.assertion.file"?: string; + + /** + * JWT audience claim. Only used when `sasl.oauthbearer.method` is set to "oidc" and JWT assertion is needed. + */ + "sasl.oauthbearer.assertion.claim.aud"?: string; + + /** + * Assertion expiration time in seconds. Only used when `sasl.oauthbearer.method` is set to "oidc" and JWT assertion is needed. + * + * @default 300 + */ + "sasl.oauthbearer.assertion.claim.exp.seconds"?: number; + + /** + * JWT issuer claim. Only used when `sasl.oauthbearer.method` is set to "oidc" and JWT assertion is needed. + */ + "sasl.oauthbearer.assertion.claim.iss"?: string; + + /** + * JWT ID claim. When set to `true`, a random UUID is generated. Only used when `sasl.oauthbearer.method` is set to "oidc" and JWT assertion is needed. + * + * @default false + */ + "sasl.oauthbearer.assertion.claim.jti.include"?: boolean; + + /** + * Assertion not before time in seconds. Only used when `sasl.oauthbearer.method` is set to "oidc" and JWT assertion is needed. + * + * @default 60 + */ + "sasl.oauthbearer.assertion.claim.nbf.seconds"?: number; + + /** + * JWT subject claim. Only used when `sasl.oauthbearer.method` is set to "oidc" and JWT assertion is needed. + */ + "sasl.oauthbearer.assertion.claim.sub"?: string; + + /** + * Path to the JWT template file. Only used when `sasl.oauthbearer.method` is set to "oidc" and JWT assertion is needed. + */ + "sasl.oauthbearer.assertion.jwt.template.file"?: string; + /** * List of plugin libraries to load (; separated). The library search path is platform dependent (see dlopen(3) for Unix and LoadLibrary() for Windows). If no filename extension is specified the platform-specific extension (such as .dll or .so) will be appended automatically. */ diff --git a/types/errors.d.ts b/types/errors.d.ts index 7f438991..250a1701 100644 --- a/types/errors.d.ts +++ b/types/errors.d.ts @@ -1,4 +1,4 @@ -// ====== Generated from librdkafka 2.10.1 file src-cpp/rdkafkacpp.h ====== +// ====== Generated from librdkafka v2.11.0-RC3 file src-cpp/rdkafkacpp.h ====== export const CODES: { ERRORS: { /* Internal errors to rdkafka: */ /** Begin internal error codes (**-200**) */