暂无图片
暂无图片
暂无图片
暂无图片
暂无图片

golang源码分析:langchaingo(1)

        本篇开始,我们开始分析langchaingo的源码,langchaingo的源码目录很复杂:
    |____CODE_OF_CONDUCT.md
    |____httputil
    | |____debug_transport.go
    | |____debug_transport_body.go
    |____go.mod
    |____tools
    | |____scraper
    | | |____scraper.go
    | | |____options.go
    | | |____doc.go
    | |____metaphor
    | | |____links.go
    | | |____documents.go
    | | |____search.go
    | | |____doc.go
    | | |____metaphor.go
    | |____wikipedia
    | | |____client.go
    | | |____doc.go
    | | |____wikipedia_test.go
    | | |____wikipedia.go
    | |____duckduckgo
    | | |____ddg.go
    | | |____internal
    | | | |____client.go
    | | |____doc.go
    | |____zapier
    | | |____zapier.go
    | | |____toolkit.go
    | | |____description.go
    | | |____internal
    | | | |____client.go
    | | | |____errors.go
    | | |____doc.go
    | | |____zapier_test.go
    | |____perplexity
    | | |____perplexity.go
    | | |____doc.go
    | | |____perplexity_test.go
    | |____calculator.go
    | |____doc.go
    | |____serpapi
    | | |____options.go
    | | |____serpapi.go
    | | |____internal
    | | | |____client.go
    | | |____doc.go
    | |____tool.go
    | |____sqldatabase
    | | |____testdata
    | | | |____db.sql
    | | |____postgresql
    | | | |____postgresql.go
    | | | |____postgresql_test.go
    | | |____sqlite3
    | | | |____sqlite3_test.go
    | | | |____sqlite3.go
    | | |____mysql
    | | | |____mysql_test.go
    | | | |____mysql.go
    | | |____sql_database.go
    |____embeddings
    | |____embedding.go
    | |____voyageai
    | | |____options.go
    | | |____voyageai.go
    | | |____voyageai_test.go
    | |____example_test.go
    | |____options.go
    | |____embedding_test.go
    | |____vertexai_palm_test.go
    | |____vector_math.go
    | |____bedrock
    | | |____options.go
    | | |____provider_cohere.go
    | | |____provider_amazon.go
    | | |____bedrock_test.go
    | | |____bedrock.go
    | |____doc.go
    | |____openai_test.go
    | |____huggingface
    | | |____options.go
    | | |____huggingface_test.go
    | | |____huggingface.go
    | |____vector_math_test.go
    | |____jina
    | | |____options.go
    | | |____jina.go
    | | |____jina_test.go
    | |____cybertron
    | | |____options.go
    | | |____cybertron.go
    | | |____cybertron_test.go
    |____memory
    | |____buffer_options.go
    | |____zep
    | | |____zep_chat_history.go
    | | |____zep_memory.go
    | | |____zep_chat_history_options.go
    | | |____zep_memory_options.go
    | |____chat_options.go
    | |____window_buffer_test.go
    | |____token_buffer_test.go
    | |____buffer_test.go
    | |____buffer.go
    | |____doc.go
    | |____sqlite3
    | | |____sqlite3_history_options.go
    | | |____sqlite3_history.go
    | | |____sqlite3_history_test.go
    | |____chat_test.go
    | |____chat.go
    | |____mongo
    | | |____mongo_chat_history_options.go
    | | |____mongo_chat_history.go
    | | |____mongo_chat_history_test.go
    | |____simple.go
    | |____token_buffer.go
    | |____window_buffer.go
    |____LICENSE
    |____callbacks
    | |____log.go
    | |____agent_final_stream.go
    | |____log_stream.go
    | |____combining.go
    | |____agent_final_stream_test.go
    | |____callbacks.go
    | |____doc.go
    | |____simple.go
    |____Makefile
    |____outputparser
    | |____combining_test.go
    | |____regex_dict.go
    | |____comma_seperated_list_test.go
    | |____structured_test.go
    | |____regex_parser_test.go
    | |____combining.go
    | |____comma_seperated_list.go
    | |____regex_dict_test.go
    | |____regex_parser.go
    | |____structured.go
    | |____doc.go
    | |____boolean_parser_test.go
    | |____defined_test.go
    | |____simple.go
    | |____defined.go
    | |____boolean_parser.go
    |____documentloaders
    | |____testdata
    | | |____sample_password.pdf
    | | |____test.csv
    | | |____test.html
    | | |____sample.pdf
    | | |____test.txt
    | |____pdf.go
    | |____html.go
    | |____text_test.go
    | |____html_test.go
    | |____text.go
    | |____csv.go
    | |____doc.go
    | |____assemblyai.go
    | |____csv_test.go
    | |____assemblyai_test.go
    | |____pdf_test.go
    | |____notion_test.go
    | |____documentloaders.go
    | |____notion.go
    |____internal
    | |____maputil
    | | |____map.go
    | |____mongodb
    | | |____client.go
    | |____imageutil
    | | |____download.go
    | |____sliceutil
    | | |____slice.go
    | | |____slice_test.go
    | |____setutil
    | | |____set.go
    |____agents
    | |____initialize.go
    | |____executor_test.go
    | |____conversational_test.go
    | |____mrkl_prompt.go
    | |____options.go
    | |____agents.go
    | |____mrkl.go
    | |____doc.go
    | |____prompts
    | | |____conversational_format_instructions.txt
    | | |____conversational_prefix.txt
    | | |____conversational_suffix.txt
    | |____executor.go
    | |____markl_test.go
    | |____conversational.go
    | |____openai_functions_agent.go
    | |____errors.go
    |____.golangci-exp.yaml
    |____go.sum
    |____docs
    | |____Makefile
    | |____docs
    | | |____getting-started
    | | | |____guide-chat.mdx
    | | | |____guide-mistral.mdx
    | | | |____guide-openai.mdx
    | | | |____guide-ollama.mdx
    | | |____index.md
    | | |____modules
    | | | |____data_connection
    | | | | |____vector_stores
    | | | | | |____index.mdx
    | | | | | |____pgvector.mdx
    | | | | |____retrievers
    | | | | | |____index.mdx
    | | | | |____text_splitters
    | | | | | |____index.mdx
    | | | | | |____examples
    | | | | | | |____index.mdx
    | | | | |____index.mdx
    | | | | |____document_loaders
    | | | | | |____index.mdx
    | | | |____memory
    | | | | |____index.mdx
    | | | | |____examples
    | | | | | |____index.mdx
    | | | |____agents
    | | | | |____tools
    | | | | | |____index.mdx
    | | | | |____index.mdx
    | | | | |____agents
    | | | | | |____index.mdx
    | | | | |____executor
    | | | | | |____index.mdx
    | | | | | |____getting-started.mdx
    | | | |____model_io
    | | | | |____index.mdx
    | | | | |____models
    | | | | | |____embeddings
    | | | | | | |____index.mdx
    | | | | | | |____integrations.mdx
    | | | | | |____chat
    | | | | | | |____index.mdx
    | | | | | | |____integrations.mdx
    | | | | | |____index.mdx
    | | | | | |____llms
    | | | | | | |____index.mdx
    | | | | | | |____Integrations
    | | | | | | | |____vertexai.mdx
    | | | | | | | |____huggingface.mdx
    | | | | | | | |____groq.mdx
    | | | | | | | |____openai.mdx
    | | | | | | | |____mistral.mdx
    | | | | | | | |____llamafile.mdx
    | | | | | | | |____fake.mdx
    | | | | | | | |____watsonx.mdx
    | | | | | | | |____local.mdx
    | | | | |____output_parsers
    | | | | | |____index.mdx
    | | | | |____prompts
    | | | | | |____index.mdx
    | | | | | |____prompt_templates
    | | | | | | |____partial_values.mdx
    | | | | | | |____index.mdx
    | | | |____chains
    | | | | |____index.mdx
    | | | | |____llm_chain.mdx
    | |____code-block-loader.js
    | |____README.md
    | |____babel.config.js
    | |____.gitignore
    | |____package-lock.json
    | |____package.json
    | |____static
    | | |____img
    | | | |____favicon-16x16.png
    | | | |____favicon.ico
    | | | |____HeliconeKeys.png
    | | | |____android-chrome-192x192.png
    | | | |____apple-touch-icon.png
    | | | |____HeliconeDashboard.png
    | | | |____android-chrome-512x512.png
    | | | |____DataberryDashboard.png
    | | | |____parrot-chainlink-icon.png
    | | | |____favicon-32x32.png
    | | | |____parrot-icon.png
    | | |____.nojekyll
    | |____.eslintrc.js
    | |____docusaurus.config.js
    | |____parity_matrix.md
    | |____sidebars.js
    | |____src
    | | |____css
    | | | |____custom.css
    | | |____theme
    | | | |____CodeBlock
    | | | | |____index.js
    | | | |____SearchBar.js
    | | |____pages
    | | | |____index.js
    |____README.md
    |____vectorstores
    | |____vectorstores.go
    | |____chroma
    | | |____options.go
    | | |____README.md
    | | |____embedder.go
    | | |____doc.go
    | | |____chroma_test.go
    | | |____chroma.go
    | |____options.go
    | |____opensearch
    | | |____opensearch.go
    | | |____opensearch_test.go
    | | |____options.go
    | | |____index_delete.go
    | | |____types.go
    | | |____index_create.go
    | | |____doc.go
    | | |____document_indexing.go
    | |____pgvector
    | | |____options.go
    | | |____pgvector_test.go
    | | |____doc.go
    | | |____pgvector.go
    | |____azureaisearch
    | | |____azureaisearch.go
    | | |____options.go
    | | |____index_retrieve.go
    | | |____index_delete.go
    | | |____types.go
    | | |____index_list.go
    | | |____helpers_http.go
    | | |____index_create.go
    | | |____document_upload.go
    | | |____doc.go
    | | |____documents_search.go
    | | |____helpers.go
    | | |____azureaisearch_test.go
    | |____pinecone
    | | |____pinecone.go
    | | |____options.go
    | | |____pinecone_test.go
    | | |____doc.go
    | |____mongovector
    | | |____option.go
    | | |____mongovector_test.go
    | | |____mock_llm.go
    | | |____doc.go
    | | |____mock_embedder.go
    | | |____mongovector.go
    | |____doc.go
    | |____qdrant
    | | |____qdrant.go
    | | |____options.go
    | | |____rest.go
    | | |____qdrant_test.go
    | | |____doc.go
    | | |____schema.go
    | |____weaviate
    | | |____weaviate_test.go
    | | |____options.go
    | | |____doc.go
    | | |____weaviate.go
    | |____milvus
    | | |____milvus.go
    | | |____options.go
    | | |____milvus_test.go
    | |____redisvector
    | | |____testdata
    | | | |____schema.yml
    | | | |____schema.json
    | | |____options.go
    | | |____redis_vector_test.go
    | | |____doc.go
    | | |____redis_client.go
    | | |____index_schema.go
    | | |____redis_vector.go
    | | |____index_schema_test.go
    | | |____index_search.go
    |____schema
    | |____memory.go
    | |____documents.go
    | |____doc.go
    | |____chat_message_history.go
    | |____output_parsers.go
    | |____schema.go
    | |____retrivers.go
    |____doc.go
    |____.gitignore
    |____jsonschema
    | |____json_test.go
    | |____json.go
    |____textsplitter
    | |____recursive_character_test.go
    | |____testdata
    | | |____example.md
    | | |____example_markdown_header_512.md
    | |____token_splitter.go
    | |____options.go
    | |____text_spliter.go
    | |____markdown_splitter_test.go
    | |____doc.go
    | |____markdown_splitter.go
    | |____recursive_character.go
    | |____token_splitter_test.go
    | |____split_documents.go
    |____CONTRIBUTING.md
    |____examples
    | |____maritaca-example
    | | |____README.md
    | | |____maritaca-chat-example.go
    | |____Makefile
    | |____zep-memory-chain-example
    | | |____README.md
    | | |____main.go
    | |____README.md
    | |____.update-all-to-latest.sh
    | |____openai-embeddings-example
    | | |____openai-embeddings-example.go
    | | |____README.md
    | |____chains-conversation-memory-sqlite
    | | |____README.md
    | | |____chains_conversation_memory_sqlite.go
    | |____deepseek-completion-example
    | | |____deepseek-completion-example.go
    |____.golangci.yaml
    |____.github
    | |____FUNDING.yml
    | |____workflows
    | | |____ci.yaml
    | | |____publish-docs.yaml
    | |____PULL_REQUEST_TEMPLATE.md
    |____prompts
    | |____prompt_test.go
    | |____prompt_template_test.go
    | |____prompts.go
    | |____few_shot.go
    | |____internal
    | | |____fstring
    | | | |____fstring.go
    | | | |____fstring_test.go
    | | | |____doc.go
    | | | |____parser.go
    | |____templates_test.go
    | |____string_prompt.go
    | |____chat_prompt_template_test.go
    | |____message_prompt_template.go
    | |____few_shot_test.go
    | |____doc.go
    | |____example_selector.go
    | |____templates.go
    | |____chat_prompt_template.go
    | |____prompt_template.go
    | |____chat_prompt.go
    |____llms
    | |____count_tokens_test.go
    | |____cohere
    | | |____internal
    | | | |____cohereclient
    | | | | |____cohereclient.go
    | | |____coherellm_option.go
    | | |____coherellm.go
    | |____count_tokens.go
    | |____generatecontent_test.go
    | |____generatecontent.go
    | |____cache
    | | |____cache_test.go
    | | |____cache.go
    | | |____inmemory
    | | | |____options.go
    | | | |____inmemory.go
    | | | |____inmemory_test.go
    | | |____doc.go
    | | |____mocks_test.go
    | |____options.go
    | |____prompts.go
    | |____marshaling_test.go
    | |____chat_messages.go
    | |____googleai
    | | |____option.go
    | | |____internal
    | | | |____cmd
    | | | | |____generate-vertex.go
    | | | |____palmclient
    | | | | |____palmclient.go
    | | |____googleai.go
    | | |____new.go
    | | |____README.md
    | | |____shared_test
    | | | |____testdata
    | | | | |____parrot-icon.png
    | | | |____shared_test.go
    | | |____embeddings.go
    | | |____vertex
    | | | |____vertex.go
    | | | |____new.go
    | | | |____embeddings.go
    | | |____palm
    | | | |____palm_llm_option.go
    | | | |____palm_llm.go
    | |____cloudflare
    | | |____options.go
    | | |____internal
    | | | |____cloudflareclient
    | | | | |____role.go
    | | | | |____client.go
    | | | | |____api.go
    | | | | |____model.go
    | | | | |____api_test.go
    | | |____cloudflarellm.go
    | |____maritaca
    | | |____maritaca_test.go
    | | |____options.go
    | | |____internal
    | | | |____maritacaclient
    | | | | |____types.go
    | | | | |____maritacaclient.go
    | | |____maritacallm.go
    | |____bedrock
    | | |____bedrockllm_test.go
    | | |____internal
    | | | |____bedrockclient
    | | | | |____provider_anthropic.go
    | | | | |____bedrockclient.go
    | | | | |____provider_cohere.go
    | | | | |____provider_amazon.go
    | | | | |____provider_meta.go
    | | | | |____provider_ai21.go
    | | |____models_list.go
    | | |____bedrockllm_option.go
    | | |____bedrockllm.go
    | |____local
    | | |____internal
    | | | |____localclient
    | | | | |____localclient.go
    | | | | |____doc.go
    | | | | |____completions.go
    | | |____localllm_option.go
    | | |____localllm.go
    | |____llms.go
    | |____llamafile
    | | |____options.go
    | | |____internal
    | | | |____llamafileclient
    | | | | |____types.go
    | | | | |____llamafileclient.go
    | | |____llamafilellm.go
    | | |____llamafilellm_test.go
    | |____watsonx
    | | |____watsonxllm.go
    | |____marshaling.go
    | |____doc.go
    | |____anthropic
    | | |____anthropicllm.go
    | | |____internal
    | | | |____anthropicclient
    | | | | |____anthropicclient.go
    | | | | |____messages.go
    | | | | |____completions.go
    | | |____anthropicllm_option.go
    | |____huggingface
    | | |____internal
    | | | |____huggingfaceclient
    | | | | |____huggingfaceclient.go
    | | | | |____inference.go
    | | | | |____embeddings.go
    | | | | |____huggingfaceclient_test.go
    | | |____huggingfacellm.go
    | | |____huggingfacellm_option.go
    | |____chat_messages_test.go
    | |____fake
    | | |____fakellm.go
    | | |____fakellm_test.go
    | |____ernie
    | | |____internal
    | | | |____ernieclient
    | | | | |____ernieclient.go
    | | | | |____ernieclient_test.go
    | | | | |____chat.go
    | | |____doc.go
    | | |____erniellm.go
    | | |____erniellm_option.go
    | |____ollama
    | | |____options.go
    | | |____ollamallm.go
    | | |____internal
    | | | |____ollamaclient
    | | | | |____types.go
    | | | | |____ollamaclient.go
    | | |____ollama_test.go
    | |____mistral
    | | |____mistralmodel.go
    | | |____mistralembed_test.go
    | | |____client_options.go
    | | |____mistralembed.go
    | |____openai
    | | |____openaillm.go
    | | |____llm.go
    | | |____internal
    | | | |____openaiclient
    | | | | |____openaiclient.go
    | | | | |____chat_test.go
    | | | | |____chat.go
    | | | | |____embeddings.go
    | | | | |____completions.go
    | | |____openaillm_option.go
    | | |____multicontent_test.go
    | | |____structured_output_test.go
    |____chains
    | |____stuff_documents.go
    | |____llm_math.go
    | |____llm_math_test.go
    | |____testdata
    | | |____mouse_story.txt
    | |____transform_test.go
    | |____refine_documents.go
    | |____chains.go
    | |____prompt_selector.go
    | |____conversational_retrieval_qa_test.go
    | |____options.go
    | |____question_answering.go
    | |____sql_database_test.go
    | |____conversational_retrieval_qa.go
    | |____llm.go
    | |____sequential_test.go
    | |____retrieval_qa.go
    | |____question_answering_test.go
    | |____conversation.go
    | |____summarization.go
    | |____constitution
    | | |____prompts.go
    | | |____principles.go
    | | |____constitutional.go
    | | |____constitutional_test.go
    | |____constitutional.go
    | |____transform.go
    | |____llm_azure_test.go
    | |____map_reduce_test.go
    | |____doc.go
    | |____map_reduce.go
    | |____stuff_documents_test.go
    | |____chains_test.go
    | |____sequential.go
    | |____api.go
    | |____map_rerank_documents_test.go
    | |____prompts
    | | |____llm_api_url_response.txt
    | | |____llm_api_url.txt
    | | |____llm_math.txt
    | |____llm_test.go
    | |____sql_database.go
    | |____map_rerank_documents.go
    | |____retrieval_qa_test.go
    | |____constitutional_test.go
    | |____summarization_test.go
    | |____conversation_test.go
    | |____api_test.go
    | |____errors.go
    |____exp
    | |____README.md
    | |____doc.go
            因此我们从最简单的提示词补全的例子开始,基于openai协议连接LLM,例子源码如下:
      package main
      import (
          "context"
          "fmt"
          "log"
          "github.com/tmc/langchaingo/llms"
          "github.com/tmc/langchaingo/llms/openai"
      )
      func main() {
          ctx := context.Background()
          llm, err := openai.New()
          if err != nil {
              log.Fatal(err)
          }
          prompt := "What would be a good company name for a company that makes colorful socks?"
          completion, err := llms.GenerateFromSinglePrompt(ctx, llm, prompt)
          if err != nil {
              log.Fatal(err)
          }
          fmt.Println(completion)
      }
          可以看到,它核心就两个调用:1,初始化LLM client,2,根据提示词获取补全后的结果
        llm, err := openai.New()
          completion, err := llms.GenerateFromSinglePrompt(ctx, llm, prompt)
                  首先我们看下如何基于openai协议进行LLM 客户端连接创建,github.com/tmc/langchaingo@v0.1.13/llms/openai/openaillm.go
            func New(opts ...Option) (*LLM, error) {
                opt, c, err := newClient(opts...)
                if err != nil {
                    return nil, err
                }
                return &LLM{
                    client:           c,
                    CallbacksHandler: opt.callbackHandler,
                }, err
            }
                    它初始化了一个连接,然后将opt的回调处理函数赋值。初始化client的代码也很简单,先从环境变量里获取token、模型名称和基础url,然后定义http客户端。接着赋值选项参数,校验参数完整性:
              func newClient(opts ...Option) (*options, *openaiclient.Client, error) {
                  options := &options{
                      token:        os.Getenv(tokenEnvVarName),
                      model:        os.Getenv(modelEnvVarName),
                      baseURL:      getEnvs(baseURLEnvVarName, baseAPIBaseEnvVarName),
                      organization: os.Getenv(organizationEnvVarName),
                      apiType:      APIType(openaiclient.APITypeOpenAI),
                      httpClient:   http.DefaultClient,
                  }
                  for _, opt := range opts {
                      opt(options)
                  }
                  // set of options needed for Azure client
                  if openaiclient.IsAzure(openaiclient.APIType(options.apiType)) && options.apiVersion == "" {
                      options.apiVersion = DefaultAPIVersion
                      if options.model == "" {
                          return options, nil, ErrMissingAzureModel
                      }
                      if options.embeddingModel == "" {
                          return options, nil, ErrMissingAzureEmbeddingModel
                      }
                  }
                  if len(options.token) == 0 {
                      return options, nil, ErrMissingToken
                  }
                  cli, err := openaiclient.New(options.token, options.model, options.baseURL, options.organization,
                      openaiclient.APIType(options.apiType), options.apiVersion, options.httpClient, options.embeddingModel,
                      options.responseFormat,
                  )
                  return options, cli, err
              }
              最后调用接口
                openaiclient.New(options.token, options.model, options.baseURL, options.organization,
                        openaiclient.APIType(options.apiType), options.apiVersion, options.httpClient, options.embeddingModel,
                        options.responseFormat,
                    )
                  func New(token string, model string, baseURL string, organization string,
                      apiType APIType, apiVersion string, httpClient Doer, embeddingModel string,
                      responseFormat *ResponseFormat,
                      opts ...Option,
                  ) (*Client, error) {
                      c := &Client{
                          token:          token,
                          Model:          model,
                          EmbeddingModel: embeddingModel,
                          baseURL:        strings.TrimSuffix(baseURL, "/"),
                          organization:   organization,
                          apiType:        apiType,
                          apiVersion:     apiVersion,
                          httpClient:     httpClient,
                          ResponseFormat: responseFormat,
                      }
                      if c.baseURL == "" {
                          c.baseURL = defaultBaseURL
                      }
                      for _, opt := range opts {
                          if err := opt(c); err != nil {
                              return nil, err
                          }
                      }
                      return c, nil
                  }
                          总结一句话,就是初始化了完整的http客户端,为后面调用接口做了准备工作;

                  文章转载自golang算法架构leetcode技术php,如果涉嫌侵权,请发送邮件至:contact@modb.pro进行举报,并提供相关证据,一经查实,墨天轮将立刻删除相关内容。

                  评论