diff --git a/lib/langchain/evals/cosine_similarity.rb b/lib/langchain/evals/cosine_similarity.rb
new file mode 100644
index 000000000..fee8445a5
--- /dev/null
+++ b/lib/langchain/evals/cosine_similarity.rb
@@ -0,0 +1,18 @@
+module Langchain
+  module Evals
+    class CosineSimilarity
+      attr_reader :llm
+
+      def initialize(llm:)
+        @llm = llm
+      end
+
+      def score(actual_output:, expected_output:)
+        vector_a = llm.embed(text: actual_output).embedding
+        vector_b = llm.embed(text: expected_output).embedding
+
+        Langchain::Utils::CosineSimilarity.new(vector_a, vector_b).calculate_similarity
+      end
+    end
+  end
+end
diff --git a/spec/langchain/evals/cosine_similarity_spec.rb b/spec/langchain/evals/cosine_similarity_spec.rb
new file mode 100644
index 000000000..999e0858e
--- /dev/null
+++ b/spec/langchain/evals/cosine_similarity_spec.rb
@@ -0,0 +1,18 @@
+RSpec.describe Langchain::Evals::CosineSimilarity do
+  let(:llm) { Langchain::LLM::OpenAI.new(api_key: "123") }
+
+  describe "#score" do
+    subject { described_class.new(llm: llm) }
+
+    let(:actual_output) { "The answer is 4" }
+    let(:expected_output) { "2 + 2 = 4" }
+
+    before do
+      allow(subject.llm).to receive(:embed).and_return(double("Langchain::LLM::OpenAIResponse", embedding: [1, 0, 0]))
+    end
+
+    it "generates the score" do
+      expect(subject.score(actual_output: actual_output, expected_output: expected_output)).to eq(1.0)
+    end
+  end
+end