Skip to content

Commit 2fce439

Browse files
heathsheaths
authored
Create end-to-end sample for an indexer (Azure#12900)
* Create end-to-end sample for an indexer Includes AI enrichments. Fixes Azure#12518 * Force line endings * Resolve PR feedback
1 parent 7813254 commit 2fce439

10 files changed

+1694
-40
lines changed

‎.gitattributes

+1-1
Original file line numberDiff line numberDiff line change
@@ -21,4 +21,4 @@
2121

2222
# Generated code
2323
**/Generated/**/*.cs linguist-generated=true
24-
**/Generated/**/*.csproj linguist-generated=true
24+
**/Generated/**/*.csproj linguist-generated=true

‎sdk/search/Azure.Search.Documents/samples/Sample01a_HelloWorld.md

+12-6
Original file line numberDiff line numberDiff line change
@@ -1,44 +1,50 @@
11
# Azure.Search.Documents Samples - Hello World (sync)
22

33
## Import the namespaces
4+
45
```C# Snippet:Azure_Search_Tests_Samples_Namespaces
56
using Azure.Search.Documents;
67
using Azure.Search.Documents.Indexes;
78
using Azure.Search.Documents.Indexes.Models;
9+
using Azure.Search.Documents.Models;
810
```
911

1012
## Create a client
13+
1114
Create a `SearchServiceClient` and send a request.
15+
1216
```C# Snippet:Azure_Search_Tests_Samples_CreateClient
1317
// Get the service endpoint and API key from the environment
1418
Uri endpoint = new Uri(Environment.GetEnvironmentVariable("SEARCH_ENDPOINT"));
1519
AzureKeyCredential credential = new AzureKeyCredential(
1620
Environment.GetEnvironmentVariable("SEARCH_API_KEY"));
1721

18-
// Create a new SearchServiceClient
19-
SearchIndexClient search = new SearchIndexClient(endpoint, credential);
22+
// Create a new SearchIndexClient
23+
SearchIndexClient indexClient = new SearchIndexClient(endpoint, credential);
2024

2125
// Perform an operation
22-
Response<SearchServiceStatistics> stats = search.GetServiceStatistics();
26+
Response<SearchServiceStatistics> stats = indexClient.GetServiceStatistics();
2327
Console.WriteLine($"You are using {stats.Value.Counters.IndexCounter.Usage} indexes.");
2428
```
2529

2630
## Handle Errors
31+
2732
All Search operations will throw a RequestFailedException on failure.
33+
2834
```C# Snippet:Azure_Search_Tests_Samples_HandleErrors
2935
Uri endpoint = new Uri(Environment.GetEnvironmentVariable("SEARCH_ENDPOINT"));
3036
AzureKeyCredential credential = new AzureKeyCredential(
3137
Environment.GetEnvironmentVariable("SEARCH_API_KEY"));
3238

3339
// Create an invalid SearchClient
3440
string fakeIndexName = "doesnotexist";
35-
SearchClient client = new SearchClient(endpoint, fakeIndexName, credential);
41+
SearchClient searchClient = new SearchClient(endpoint, fakeIndexName, credential);
3642
try
3743
{
38-
client.GetDocumentCount();
44+
searchClient.GetDocumentCount();
3945
}
4046
catch (RequestFailedException ex) when (ex.Status == 404)
4147
{
4248
Console.WriteLine("Index wasn't found.");
4349
}
44-
```
50+
```

‎sdk/search/Azure.Search.Documents/samples/Sample01b_HelloWorldAsync.md

+12-6
Original file line numberDiff line numberDiff line change
@@ -1,44 +1,50 @@
11
# Azure.Search.Documents Samples - Hello World (async)
22

33
## Import the namespaces
4+
45
```C# Snippet:Azure_Search_Tests_Samples_Namespaces
56
using Azure.Search.Documents;
67
using Azure.Search.Documents.Indexes;
78
using Azure.Search.Documents.Indexes.Models;
9+
using Azure.Search.Documents.Models;
810
```
911

1012
## Create a client
13+
1114
Create a `SearchServiceClient` and send a request.
15+
1216
```C# Snippet:Azure_Search_Tests_Samples_CreateClientAsync
1317
// Get the service endpoint and API key from the environment
1418
Uri endpoint = new Uri(Environment.GetEnvironmentVariable("SEARCH_ENDPOINT"));
1519
AzureKeyCredential credential = new AzureKeyCredential(
1620
Environment.GetEnvironmentVariable("SEARCH_API_KEY"));
1721

18-
// Create a new SearchServiceClient
19-
SearchIndexClient search = new SearchIndexClient(endpoint, credential);
22+
// Create a new SearchIndexClient
23+
SearchIndexClient indexClient = new SearchIndexClient(endpoint, credential);
2024

2125
// Perform an operation
22-
Response<SearchServiceStatistics> stats = await search.GetServiceStatisticsAsync();
26+
Response<SearchServiceStatistics> stats = await indexClient.GetServiceStatisticsAsync();
2327
Console.WriteLine($"You are using {stats.Value.Counters.IndexCounter.Usage} indexes.");
2428
```
2529

2630
## Handle Errors
31+
2732
All Search operations will throw a RequestFailedException on failure.
33+
2834
```C# Snippet:Azure_Search_Tests_Samples_HandleErrorsAsync
2935
Uri endpoint = new Uri(Environment.GetEnvironmentVariable("SEARCH_ENDPOINT"));
3036
AzureKeyCredential credential = new AzureKeyCredential(
3137
Environment.GetEnvironmentVariable("SEARCH_API_KEY"));
3238

3339
// Create an invalid SearchClient
3440
string fakeIndexName = "doesnotexist";
35-
SearchClient client = new SearchClient(endpoint, fakeIndexName, credential);
41+
SearchClient searchClient = new SearchClient(endpoint, fakeIndexName, credential);
3642
try
3743
{
38-
await client.GetDocumentCountAsync();
44+
await searchClient.GetDocumentCountAsync();
3945
}
4046
catch (RequestFailedException ex) when (ex.Status == 404)
4147
{
4248
Console.WriteLine("Index wasn't found.");
4349
}
44-
```
50+
```

‎sdk/search/Azure.Search.Documents/samples/Sample02_Service.md

+233-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# Azure.Search.Documents Samples - Service Operations
22

33
## Get Statistics
4+
45
Gets service level statistics for a Search Service.
56

67
This operation returns the number and type of objects in your service, the
@@ -13,13 +14,242 @@ not in real time. Therefore, the statistics returned by this API may not
1314
reflect changes caused by recent indexing operations.
1415

1516
```C# Snippet:Azure_Search_Tests_Samples_GetStatisticsAsync
16-
// Create a new SearchServiceClient
17+
// Create a new SearchIndexClient
1718
Uri endpoint = new Uri(Environment.GetEnvironmentVariable("SEARCH_ENDPOINT"));
1819
AzureKeyCredential credential = new AzureKeyCredential(
1920
Environment.GetEnvironmentVariable("SEARCH_API_KEY"));
20-
SearchIndexClient search = new SearchIndexClient(endpoint, credential);
21+
SearchIndexClient indexClient = new SearchIndexClient(endpoint, credential);
2122

2223
// Get and report the Search Service statistics
23-
Response<SearchServiceStatistics> stats = await search.GetServiceStatisticsAsync();
24+
Response<SearchServiceStatistics> stats = await indexClient.GetServiceStatisticsAsync();
2425
Console.WriteLine($"You are using {stats.Value.Counters.IndexCounter.Usage} of {stats.Value.Counters.IndexCounter.Quota} indexes.");
2526
```
27+
28+
## Connect a Data Source to an Index
29+
30+
You can index documents already stored in Azure Blob Storage, Azure SQL, Azure Table Storage, Azure Cosmos DB, or MySQL.
31+
This not only provides you flexibility to use existing storage, but to also reduce the index size by only defining the
32+
fields you need in the index and mapping just those fields from your data source. An indexer is responsible for using
33+
those mappings and running optional skillsets to populate your index.
34+
35+
The following sample will index hotel information from Azure Blob Storage and automatically translate English descriptions
36+
to French if not a human translation is not already defined.
37+
38+
### Create a Synonym Map
39+
40+
First we'll create a synonym map for country names and abbreviations. A synonym map contains aliases
41+
and other transformations using the
42+
[Solr format](https://docs.microsoft.com/rest/api/searchservice/create-synonym-map#apache-solr-synonym-format),
43+
for example:
44+
45+
```
46+
United States of America, US, USA
47+
Washington, Wash. => WA
48+
```
49+
50+
You can pass a preformatted string delimited by `\n` characters, or a `TextReader` for
51+
a file you may have downloaded or have stored elsewhere.
52+
53+
```C# Snippet:Azure_Search_Tests_Samples_CreateIndexerAsync_CreateSynonymMap
54+
// Create a new SearchIndexClient
55+
Uri endpoint = new Uri(Environment.GetEnvironmentVariable("SEARCH_ENDPOINT"));
56+
AzureKeyCredential credential = new AzureKeyCredential(
57+
Environment.GetEnvironmentVariable("SEARCH_API_KEY"));
58+
SearchIndexClient indexClient = new SearchIndexClient(endpoint, credential);
59+
60+
// Create a synonym map from a file containing country names and abbreviations
61+
// using the Solr format with entry on a new line using \n, for example:
62+
// United States of America,US,USA\n
63+
string synonymMapName = "countries";
64+
string synonymMapPath = "countries.txt";
65+
66+
SynonymMap synonyms;
67+
using (StreamReader file = File.OpenText(synonymMapPath))
68+
{
69+
synonyms = new SynonymMap(synonymMapName, file);
70+
}
71+
72+
await indexClient.CreateSynonymMapAsync(synonyms);
73+
```
74+
75+
These synonym maps can be associated with search fields so users can query for terms
76+
with which they may be more familiar. These will be referenced further below when we
77+
create an index.
78+
79+
### Create an Index
80+
81+
Next we'll create an index of hotel information that uses the synonym map we created above
82+
attached to the `country` search field:
83+
84+
```C# Snippet:Azure_Search_Tests_Samples_CreateIndexerAsync_CreateIndex
85+
// Create the index
86+
string indexName = "hotels";
87+
SearchIndex index = new SearchIndex(indexName)
88+
{
89+
Fields =
90+
{
91+
new SimpleField("hotelId", SearchFieldDataType.String) { IsKey = true, IsFilterable = true, IsSortable = true },
92+
new SearchableField("hotelName") { IsFilterable = true, IsSortable = true },
93+
new SearchableField("description") { AnalyzerName = LexicalAnalyzerName.EnLucene },
94+
new SearchableField("descriptionFr") { AnalyzerName = LexicalAnalyzerName.FrLucene },
95+
new SearchableField("tags", collection: true) { IsFilterable = true, IsFacetable = true },
96+
new ComplexField("address")
97+
{
98+
Fields =
99+
{
100+
new SearchableField("streetAddress"),
101+
new SearchableField("city") { IsFilterable = true, IsSortable = true, IsFacetable = true },
102+
new SearchableField("stateProvince") { IsFilterable = true, IsSortable = true, IsFacetable = true },
103+
new SearchableField("country") { SynonymMapNames = new[] { synonymMapName }, IsFilterable = true, IsSortable = true, IsFacetable = true },
104+
new SearchableField("postalCode") { IsFilterable = true, IsSortable = true, IsFacetable = true }
105+
}
106+
}
107+
}
108+
};
109+
110+
await indexClient.CreateIndexAsync(index);
111+
```
112+
113+
### Create a Data Source Connection
114+
115+
We'll need to create a connection to Azure Blob Storage where our documents are found.
116+
These JSON documents match the schema of our index, but you can map input and output fields to change names
117+
when indexing to better fit your index. You might do this, for example, if you have multiple indexers
118+
indexing data from different data sources with different schemas and data.
119+
120+
```C# Snippet:Azure_Search_Tests_Samples_CreateIndexerAsync_CreateDataSourceConnection
121+
// Create a new SearchIndexerClient
122+
SearchIndexerClient indexerClient = new SearchIndexerClient(endpoint, credential);
123+
124+
string dataSourceConnectionName = "hotels";
125+
SearchIndexerDataSourceConnection dataSourceConnection = new SearchIndexerDataSourceConnection(
126+
dataSourceConnectionName,
127+
SearchIndexerDataSourceType.AzureBlob,
128+
Environment.GetEnvironmentVariable("STORAGE_CONNECTION_STRING"),
129+
new SearchIndexerDataContainer(Environment.GetEnvironmentVariable("STORAGE_CONTAINER")));
130+
131+
await indexerClient.CreateDataSourceConnectionAsync(dataSourceConnection);
132+
```
133+
134+
### Create a Skillset
135+
136+
To provide French translations of descriptions, we'll define a [translation skill](https://docs.microsoft.com/azure/search/cognitive-search-skill-text-translation) to translate from English.
137+
We'll also define a [conditional skill](https://docs.microsoft.com/azure/search/cognitive-search-skill-conditional) to use a human-translated descriptions instead if available.
138+
139+
See all [built-in skills](https://docs.microsoft.com/azure/search/cognitive-search-predefined-skills) for more information
140+
about all available skills.
141+
142+
```C# Snippet:Azure_Search_Tests_Samples_CreateIndexerAsync_Skillset
143+
// Translate English descriptions to French.
144+
// See https://docs.microsoft.com/azure/search/cognitive-search-skill-text-translation for details of the Text Translation skill.
145+
TextTranslationSkill translationSkill = new TextTranslationSkill(
146+
inputs: new[]
147+
{
148+
new InputFieldMappingEntry("text") { Source = "/document/description" }
149+
},
150+
outputs: new[]
151+
{
152+
new OutputFieldMappingEntry("translatedText") { TargetName = "descriptionFrTranslated" }
153+
},
154+
TextTranslationSkillLanguage.Fr)
155+
{
156+
Name = "descriptionFrTranslation",
157+
Context = "/document",
158+
DefaultFromLanguageCode = TextTranslationSkillLanguage.En
159+
};
160+
161+
// Use the human-translated French description if available; otherwise, use the translated description.
162+
// See https://docs.microsoft.com/azure/search/cognitive-search-skill-conditional for details of the Conditional skill.
163+
ConditionalSkill conditionalSkill = new ConditionalSkill(
164+
inputs: new[]
165+
{
166+
new InputFieldMappingEntry("condition") { Source = "= $(/document/descriptionFr) == null" },
167+
new InputFieldMappingEntry("whenTrue") { Source = "/document/descriptionFrTranslated" },
168+
new InputFieldMappingEntry("whenFalse") { Source = "/document/descriptionFr" }
169+
},
170+
outputs: new[]
171+
{
172+
new OutputFieldMappingEntry("output") { TargetName = "descriptionFrFinal"}
173+
})
174+
{
175+
Name = "descriptionFrConditional",
176+
Context = "/document",
177+
};
178+
179+
// Create a SearchIndexerSkillset that processes those skills in the order given below.
180+
string skillsetName = "translations";
181+
SearchIndexerSkillset skillset = new SearchIndexerSkillset(
182+
skillsetName,
183+
new SearchIndexerSkill[] { translationSkill, conditionalSkill })
184+
{
185+
CognitiveServicesAccount = new CognitiveServicesAccountKey(Environment.GetEnvironmentVariable("COGNITIVE_KEY"))
186+
};
187+
188+
await indexerClient.CreateSkillsetAsync(skillset);
189+
```
190+
191+
### Create an Indexer
192+
193+
Finally we'll create an indexer to index the documents from Azure Blob Storage and process our skillset to translate the
194+
English description to French if required. We also have to tell the indexer to process each blob as a separate JSON document
195+
to maintain the document structure required for skills.
196+
197+
```C# Snippet:Azure_Search_Tests_Samples_CreateIndexerAsync_CreateIndexer
198+
string indexerName = "hotels";
199+
SearchIndexer indexer = new SearchIndexer(
200+
indexerName,
201+
dataSourceConnectionName,
202+
indexName)
203+
{
204+
// We only want to index fields defined in our index, excluding descriptionFr if defined.
205+
FieldMappings =
206+
{
207+
new FieldMapping("hotelId"),
208+
new FieldMapping("hotelName"),
209+
new FieldMapping("description"),
210+
new FieldMapping("tags"),
211+
new FieldMapping("address")
212+
},
213+
OutputFieldMappings =
214+
{
215+
new FieldMapping("/document/descriptionFrFinal") { TargetFieldName = "descriptionFr" }
216+
},
217+
Parameters = new IndexingParameters
218+
{
219+
// Tell the indexer to parse each blob as a separate JSON document.
220+
// See https://docs.microsoft.com/azure/search/search-howto-index-json-blobs for details.
221+
Configuration =
222+
{
223+
["parsingMode"] = "json"
224+
}
225+
},
226+
SkillsetName = skillsetName
227+
};
228+
229+
// Create the indexer which, upon successful creation, also runs the indexer.
230+
await indexerClient.CreateIndexerAsync(indexer);
231+
```
232+
233+
### Querying the Index
234+
235+
Let's query the index and make sure everything works as implemented. Within the test data, hotel "6" has a nice ocean view
236+
but was not authored with a French description.
237+
238+
```C# Snippet:Azure_Search_Tests_Samples_CreateIndexerAsync_Query
239+
// Get a SearchClient from the SearchIndexClient to share its pipeline.
240+
SearchClient searchClient = indexClient.GetSearchClient(indexName);
241+
242+
// Query for hotels with an ocean view.
243+
SearchResults<Hotel> results = await searchClient.SearchAsync<Hotel>("ocean view");
244+
bool found = false;
245+
await foreach (SearchResult<Hotel> result in results.GetResultsAsync())
246+
{
247+
Hotel hotel = result.Document;
248+
249+
Console.WriteLine($"{hotel.HotelName} ({hotel.HotelId})");
250+
Console.WriteLine($" Description (English): {hotel.Description}");
251+
Console.WriteLine($" Description (French): {hotel.DescriptionFr}");
252+
}
253+
```
254+
255+
You should see within your results that hotel with a French description translated from the skill we added.

‎sdk/search/Azure.Search.Documents/samples/Sample03_Index.md

+4-2
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,18 @@
11
# Azure.Search.Documents Samples - Index Operations
22

33
## Get Count
4+
45
Retrieve a count of the number of documents in this search index.
6+
57
```C# Snippet:Azure_Search_Tests_Samples_GetCountAsync
68
// Create a SearchClient
79
Uri endpoint = new Uri(Environment.GetEnvironmentVariable("SEARCH_ENDPOINT"));
810
AzureKeyCredential credential = new AzureKeyCredential(
911
Environment.GetEnvironmentVariable("SEARCH_API_KEY"));
1012
string indexName = Environment.GetEnvironmentVariable("SEARCH_INDEX");
11-
SearchClient client = new SearchClient(endpoint, indexName, credential);
13+
SearchClient searchClient = new SearchClient(endpoint, indexName, credential);
1214

1315
// Get and report the number of documents in the index
14-
Response<long> count = await client.GetDocumentCountAsync();
16+
Response<long> count = await searchClient.GetDocumentCountAsync();
1517
Console.WriteLine($"Search index {indexName} has {count.Value} documents.");
1618
```

0 commit comments

Comments
 (0)