Enterprise Search Engines Java Clients

Table of Contents

Elasticsearch - Java clients
- Elasticsearch clients
OpenSearch Java Clients
- spring-data-opensearch
- Opensearch clients
Tags

Elasticsearch - Java clients

Trouble with these Java clients is, when organizations decide to make changes like stop using Elasticsearch and start using OpenSearch instead, if these Java clients do not support OpenSearch, the client code in the applications to work with Elasticsearch needs to be re-written to work with OpenSearch.

Elasticsearch clients

For Java applications including Spring Boot applications, Elasticsearch provides the following clients for integration:

Java Transport Client

Deprecated in Elasticsearch 7.0.0.
Provides a client object to execute all operations asynchronously, accepting a listener or returning a future.

Java REST Client

Composed of the Low Level REST Client and the High Level REST Client.
1. The Low Level Client provides load balancing, failover, persistent connections, and request/response trace logging.
2. The High Level Client works on top of the Low Level Client and is the replacement for the TransportClient.
It depends on the Elasticsearch core and provides synchronous and asynchronous APIs.
Deprecated in 7.15.0.
This is deprecated in favour of the Java API Client.

Java API Client

The new client library, independent of Elasticsearch core, provides strongly typed requests and responses, blocking and asynchronous versions for all APIs, fluent builders and functional patterns, as well as jackson and JSON-b support.
Since 7.15
https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/7.17/index.html
Migrating from the High Level Rest Client: https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/7.17/migrate-hlrc.html

Jest framework

https://github.com/searchbox-io/Jest
This was used by a lot of applications to connect to Elasticsearch when organizations were actively using Elasticsearch (even if was with AWS provider. AWS made the decision to stop supporting Elasticsearch in favor of OpenSearch much recently).
https://github.com/searchbox-io/Jest is archived. From their Github page, it doesn’t look like their latest release which is 6.3.1 https://github.com/searchbox-io/Jest/releases, is compitable with elasticsearch 7.X.
This may not be a good fit for most recent versions of Elasticsearch or Opensearch.
For these reasons, it is not a good idea to use this library for new applications.

I tried to make the existing Jest configuration work with opensearch by trying this. It did not work.

import com.google.gson.Gson;
import com.google.gson.GsonBuilder;

import io.searchbox.client.JestClient;
import io.searchbox.client.JestClientFactory;
import io.searchbox.client.JestResult;
import io.searchbox.client.config.HttpClientConfig;
import io.searchbox.core.Bulk;
import io.searchbox.core.Index;

import lombok.Data;
import lombok.extern.slf4j.Slf4j;

import org.apache.http.HttpRequestInterceptor;
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.auth.signer.Aws4Signer;
import software.amazon.awssdk.regions.Region;

import javax.annotation.PostConstruct;
import java.util.Date;
import java.util.List;

@Component
@Data
@Slf4j
public class ElasticSearchImpl {

  private JestClient client;

  private JestClient expClient;

  @Autowired
  private JestInitParams jestInitParams;

  @PostConstruct
  void init() {
          try {
                  {
                          Gson gson = new GsonBuilder().registerTypeAdapter(Date.class, new DateTimeTypeConverter()).create();
                          HttpClientConfig httpClientConfig = new HttpClientConfig.Builder(jestInitParams.getUrl())
                                          .gson(gson)
                                          .multiThreaded(true)
                                          .defaultMaxTotalConnectionPerRoute(jestInitParams.getRptRecptDefaultMaxTotalConnectionPerRoute())
                                          .maxTotalConnection(jestInitParams.getRptRecptMaxTotalConnection()).readTimeout(6000000)
                                          .build();

                          // Jest client factory
                          JestClientFactory factory = getJestClientFactory();
                          factory.setHttpClientConfig(httpClientConfig);
                          client = factory.getObject();
                  }
                  {
                          Gson gson = new GsonBuilder().registerTypeAdapter(Date.class, new ExpDateTimeTypeConverter()).create();
                          HttpClientConfig httpClientConfig = new HttpClientConfig.Builder(jestInitParams.getUrl()).gson(gson)
                                          .multiThreaded(true)
                                          .defaultMaxTotalConnectionPerRoute(jestInitParams.getExpDefaultMaxTotalConnectionPerRoute())
                                          .maxTotalConnection(jestInitParams.getExpMaxTotalConnection()).readTimeout(6000000)
                                          .build();

                          // Jest client factory
                          JestClientFactory factory = getJestClientFactory();
                          factory.setHttpClientConfig(httpClientConfig);
                          expClient = factory.getObject();
                  }
          } catch (Exception e) {
                  throw new RuntimeException(e);
          }
  }

  private static JestClientFactory getJestClientFactory() {

          String accessKey = "an-access-key";
          String secretKey = "a-secret-key";
          AwsBasicCredentials basicAWSCredentials = AwsBasicCredentials.builder().accessKeyId(accessKey).secretAccessKey(secretKey).build();
          StaticCredentialsProvider staticCredentialsProvider = StaticCredentialsProvider.create(basicAWSCredentials);

          String serviceName = "es";
          HttpRequestInterceptor interceptor = new AwsRequestSigningApacheInterceptor(
                          serviceName,
                          Aws4Signer.create(),
                          staticCredentialsProvider,
                          Region.US_EAST_1);

          JestClientFactory factory = new JestClientFactory() {

                  @Override
                  protected HttpAsyncClientBuilder configureHttpClient(HttpAsyncClientBuilder builder) {
                          return builder.addInterceptorLast(interceptor);
                  }
          };
          return factory;
  }
}

The error was:

org.springframework.beans.factory.BeanCreationException: Invocation of init method failed; nested exception is java.io.IOException: Request PUT https://vpc-address-for-opensearch.us-east-1.es.amazonaws.com/<index-name> HTTP/1.1 yielded text/plain;charset=UTF-8, should be json: HTTP/1.1 401 Unauthorized

OpenSearch Java Clients

spring-data-opensearch

https://github.com/opensearch-project/spring-data-opensearch

Opensearch clients

OpenSearch Java high-level REST client

Note: Do not use this.

According to OpenSearch documentation, The OpenSearch Java high-level REST client is deprecated. Support will be removed in OpenSearch version 3.0.0. We recommend switching to the Java client instead.

AWSJDK opensearch

Note: Do not use these.

AWSJDK opensearch see https://docs.aws.amazon.com/opensearch-service/latest/developerguide/configuration-samples.html and https://mvnrepository.com/artifact/software.amazon.awssdk/opensearch

According to the documentation here https://docs.aws.amazon.com/opensearch-service/latest/developerguide/samplecode.html: These client versions are out of date and are not updated with the latest dependencies, including Log4j. We highly recommend using the OpenSearch versions of the clients when possible.

Another library from aws: https://github.com/aws-samples/opensearch-bootful

opensearch-java client

https://opensearch.org/docs/latest/clients/java/

This seems to be the most recommended library for working with opensearch

opensearch-java see https://opensearch.org/docs/latest/clients/java/ and https://mvnrepository.com/artifact/org.opensearch.client/opensearch-java

See https://github.com/explorer436/programming-playground/tree/main/java-playground/opensearch-java-demo

When working with an AWS environment, the OpensearchClientCreator looks different

Credentials: (Development in local machines) Make sure that the access key and secret key that the Java application is using are correct and match the credentials in the AWS profile. Configure them in ~/.aws/credentials using aws cli.
Permissions: (Deployed environments)
1. Make sure that the IAM user/role associated with your credentials has the necessary permissions to interact with your OpenSearch domain.
2. Make sure that the opensearch domain configuration is updated to grant permissions to the application’s role appropriately (read, write, or all_access).
3. Both of these need to be enabled for the successful connection between the application and opensearch.
Region: Make sure that your Java application is using the same region as your OpenSearch domain.
Endpoint: Make sure that your Java application is using the correct endpoint URL which excludes https://.
Version: Make sure that your Java application implemented correct version of the OpenSearchClient for the version of OpenSearch you are using.

import lombok.extern.slf4j.Slf4j;
import org.opensearch.client.opensearch.OpenSearchClient;
import org.opensearch.client.transport.aws.AwsSdk2Transport;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import org.opensearch.client.transport.aws.AwsSdk2TransportOptions;
import org.springframework.util.ObjectUtils;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.http.apache.ApacheHttpClient;

import java.net.MalformedURLException;
import java.net.URL;

@Component
@Slf4j
public class OpensearchClientCreator {

    @Value("${opensearch.url}")
    private String opensearchUrl;

    public OpenSearchClient client;

    public OpenSearchClient getClient() throws MalformedURLException {
        log.info(">>> get OpenSearchClient...");
        if (ObjectUtils.isEmpty(client)) {
            client = createClient();
        }
        return client;
    }

    private OpenSearchClient createClient() throws MalformedURLException {

        log.info(">>> create OpenSearchClient...");

        software.amazon.awssdk.http.SdkHttpClient httpClient = ApacheHttpClient.builder().build();

        URL url = new URL(opensearchUrl);
        log.info("url: {}", url);

        String protocol = url.getProtocol();
        log.info("protocol: {}", protocol);

        String host = opensearchUrl.replaceFirst(protocol + "://", "");
        log.info("host: {}", host);

            client = new OpenSearchClient(
                    new AwsSdk2Transport(
                            httpClient,
                            host, // OpenSearch endpoint, without https://
                            "es",
                            Region.US_EAST_1,
                            AwsSdk2TransportOptions.builder().build()
                    )
            );

        return client;
    }
}