xapian-bindings/ruby/smoketest.rb

   1 #!/usr/bin/ruby -w
   2 #
   3 # smoketest.rb - test Xapian bindings for Ruby
   4 # Original version by Paul Legato (plegato@nks.net), 4/17/2006
   5 #
   6 # Originally based on smoketest.php from the PHP4 bindings.
   7 #
   8 # Copyright (C) 2006 Networked Knowledge Systems, Inc.
   9 # Copyright (C) 2008,2009,2010,2011,2016,2017,2019 Olly Betts
  10 # Copyright (C) 2010 Richard Boulton
  11 #
  12 # This program is free software; you can redistribute it and/or
  13 # modify it under the terms of the GNU General Public License as
  14 # published by the Free Software Foundation; either version 2 of the
  15 # License, or (at your option) any later version.
  16 #
  17 # This program is distributed in the hope that it will be useful,
  18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 # GNU General Public License for more details.
  21 #
  22 # You should have received a copy of the GNU General Public License
  23 # along with this program; if not, write to the Free Software
  24 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  25 # USA
  26
  27 require 'test/unit'
  28 require 'tmpdir'
  29 require 'xapian'
  30
  31 class TestMatchDecider < Xapian::MatchDecider
  32   def __call__(doc)
  33     return doc.value(0) == "yes"
  34   end
  35 end
  36
  37 class XapianSmoketest < Test::Unit::TestCase
  38
  39   def setup
  40     @stem = Xapian::Stem.new("english")
  41
  42     @doc = Xapian::Document.new()
  43     @doc.data = "is there anybody out there?"
  44     @doc.add_posting(@stem.call("is"), 1)
  45     @doc.add_posting(@stem.call("there"), 2)
  46     @doc.add_posting(@stem.call("anybody"), 3)
  47     @doc.add_posting(@stem.call("out"), 4)
  48     @doc.add_posting(@stem.call("there"), 5)
  49     @doc.add_term("XYzzy")
  50
  51     @db = Xapian::WritableDatabase.new("", Xapian::DB_BACKEND_INMEMORY)
  52     @db.add_document(@doc)
  53
  54     @enq = Xapian::Enquire.new(@db)
  55
  56     # Check Xapian::BAD_VALUENO is wrapped suitably.
  57     @enq.collapse_key = Xapian::BAD_VALUENO
  58
  59     # Test that the non-constant wrapping prior to 1.4.10 still works.
  60     @enq.collapse_key = Xapian::BAD_VALUENO()
  61   end # setup
  62
  63   def test_version
  64     # Test the version number reporting functions give plausible results.
  65     @v = sprintf("%d.%d.%d", Xapian::major_version(), Xapian::minor_version(),
  66                  Xapian::revision())
  67     @v2 = Xapian::version_string()
  68     assert_equal(@v2, @v)
  69   end # test_version
  70
  71   def test_stem
  72     assert_equal("Xapian::Stem(english)", @stem.description())
  73
  74     assert_equal("is", @stem.call("is"))
  75     assert_equal("go", @stem.call("going"))
  76     assert_equal("want", @stem.call("wanted"))
  77     assert_equal("refer", @stem.call("reference"))
  78   end # test_stem
  79
  80   # subtests are those on which some test_foo() method depends.
  81   def test_000_document
  82     assert_not_nil(@doc)
  83
  84     assert_equal("is there anybody out there?", @doc.data())
  85
  86     assert_equal(@doc.termlist_count(), 5)
  87     assert_equal("XYzzy", @doc.terms().first.term)
  88
  89     @doc.add_term("foo")
  90     assert_equal(6, @doc.termlist_count())
  91     assert_equal(@doc.terms.size(), @doc.termlist_count())
  92
  93   end # test_document
  94
  95   def test_001_database
  96     assert_not_nil(@db)
  97     assert_equal("WritableDatabase(InMemory)", @db.description())
  98     assert_equal(1, @db.doccount())
  99   end # test_database
 100
 101   def test_002_queries
 102     assert_equal("Query((smoke OR test OR terms))",
 103                  Xapian::Query.new(Xapian::Query::OP_OR, ["smoke", "test", "terms"]).description())
 104
 105     phrase_query = Xapian::Query.new(Xapian::Query::OP_PHRASE, ["smoke", "test", "tuple"])
 106     xor_query = Xapian::Query.new(Xapian::Query::OP_XOR, [ Xapian::Query.new("smoke"), phrase_query, "string" ])
 107
 108     assert_equal("Query((smoke PHRASE 3 test PHRASE 3 tuple))", phrase_query.description())
 109     assert_equal("Query((smoke XOR (smoke PHRASE 3 test PHRASE 3 tuple) XOR string))", xor_query.description())
 110
 111     assert_equal([Xapian::Term.new("smoke", 1),
 112                   Xapian::Term.new("string", 1),
 113                   Xapian::Term.new("test", 1),
 114                   Xapian::Term.new("tuple", 1)], xor_query.terms())
 115     assert_equal(xor_query.terms(), xor_query.unique_terms())
 116
 117     non_unique_query = Xapian::Query.new(Xapian::Query::OP_PHRASE,
 118                                          [Xapian::Query.new("the", 1, 1),
 119                                           Xapian::Query.new("the", 1, 2)])
 120     assert_equal([Xapian::Term.new("the", 1),
 121                   Xapian::Term.new("the", 1)], non_unique_query.terms())
 122     assert_equal([Xapian::Term.new("the", 1)], non_unique_query.unique_terms())
 123
 124     count = 0
 125     non_unique_query.terms() { count += 1 }
 126     assert_equal(2, count)
 127
 128     count = 0
 129     non_unique_query.unique_terms() { count += 1 }
 130     assert_equal(1, count)
 131
 132     assert_equal(Xapian::Query::OP_ELITE_SET, 10)
 133
 134     assert_equal("Query(<alldocuments>)", Xapian::Query::MatchAll.description())
 135     assert_equal("Query()", Xapian::Query::MatchNothing.description())
 136   end # test_queries
 137
 138   def test_003_enquire
 139     @enq = Xapian::Enquire.new(@db)
 140     assert_not_nil(@enq)
 141
 142     @enq.query = Xapian::Query.new(Xapian::Query::OP_OR, "there", "is")
 143     mset = @enq.mset(0, 10)
 144
 145     assert_equal(1, mset.size())
 146
 147     # Feature test for Enquire.matching_terms()
 148     assert_equal(2, @enq.matching_terms(mset.hit(0)).size())
 149     assert_equal([Xapian::Term.new("is", 1), Xapian::Term.new("there", 1)],
 150                  @enq.matching_terms(mset.hit(0)))
 151   end # test_enquire
 152
 153   def test_004_mset_iterator
 154     @enq = Xapian::Enquire.new(@db)
 155     assert_not_nil(@enq)
 156
 157     @enq.query = Xapian::Query.new(Xapian::Query::OP_OR, "there", "is")
 158     mset = @enq.mset(0, 10)
 159
 160     assert_equal(mset.matches().size(), mset.size())
 161     msize = 0
 162     mset.matches { |x| msize += 1 }
 163     assert_equal(msize, mset.size())
 164   end
 165
 166
 167   def test_005_eset_iterator
 168     rset = Xapian::RSet.new
 169
 170     rset.add_document(1)
 171
 172     @enq = Xapian::Enquire.new(@db)
 173     @enq.query = Xapian::Query.new(Xapian::Query::OP_OR, "there", "is")
 174
 175     eset = @enq.eset(10, rset)
 176     assert_not_nil(eset)
 177
 178     assert_equal(3, eset.terms.size())
 179     assert_equal(3, eset.size())
 180     esize = 0
 181     eset.terms { |x| esize += 1 }
 182     assert_equal(3, esize)
 183   end # test_eset_iter
 184
 185   # Feature test for Database.allterms
 186   def test_006_database_allterms
 187     assert_equal(5, @db.allterms.size())
 188     ou_terms = @db.allterms('ou')
 189     assert_equal(1, ou_terms.size())
 190     assert_equal('out', ou_terms[0].term)
 191     count = 0
 192     @db.allterms('ou') { |t|
 193       count += 1
 194       assert_equal(t.term, "out")
 195     }
 196     assert_equal(1, count)
 197   end
 198
 199   # Feature test for Database.postlist
 200   def test_007_database_postlist
 201     assert_equal(1, @db.postlist("there").size())
 202     count = 0
 203     @db.postlist("there") { |x| count += 1 }
 204     assert_equal(1, count)
 205   end
 206
 207   # Feature test for Database.termlist
 208   def test_008_database_termlist
 209     assert_equal(5, @db.termlist(1).size())
 210     count = 0
 211     @db.termlist(1) { |t| count += 1 }
 212     assert_equal(5, count)
 213   end
 214
 215   # Feature test for Database.positionlist
 216   def test_009_database_positionlist
 217     assert_equal(2, @db.positionlist(1, "there").size())
 218     count = 0
 219     @db.positionlist(1, "there") { |x| count += 1 }
 220     assert_equal(2, count)
 221   end
 222
 223   # Feature test for Document.values
 224   def test_010_document_values
 225     assert_equal(0, @doc.values().size())
 226     @doc.values() { |x| assert(false) }
 227   end
 228
 229   def test_011_matchdecider
 230     @doc = Xapian::Document.new()
 231     @doc.data = "Two"
 232     @doc.add_posting(@stem.call("out"), 1)
 233     @doc.add_posting(@stem.call("source"), 2)
 234     @doc.add_value(0, "yes")
 235     @db.add_document(@doc)
 236
 237     @query = Xapian::Query.new(@stem.call("out"))
 238     enquire = Xapian::Enquire.new(@db)
 239     enquire.query = @query
 240     mset = enquire.mset(0, 10, nil, TestMatchDecider.new)
 241     assert_equal(mset.size(), 1)
 242     assert_equal(mset.docid(0), 2)
 243   end
 244
 245   def test_012_metadata
 246     assert_equal(@db.get_metadata('Foo'), '')
 247     @db.set_metadata('Foo', 'Foo')
 248     assert_equal(@db.get_metadata('Foo'), 'Foo')
 249
 250     # The inmemory backend doesn't support metadata_keys so we need to create a
 251     # "real" database for these tests.
 252     Dir.mktmpdir("smokerb") {|tmpdir|
 253         dbpath = "#{tmpdir}/db"
 254
 255         db = Xapian::WritableDatabase.new(dbpath, Xapian::DB_CREATE_OR_OVERWRITE)
 256         assert_equal(db.get_metadata('Foo'), '')
 257         db.set_metadata('Foo', 'Foo')
 258         assert_equal(db.get_metadata('Foo'), 'Foo')
 259         assert_equal(db.metadata_keys(), ["Foo"])
 260         assert_equal(db.metadata_keys('F'), ["Foo"])
 261         assert_equal(db.metadata_keys('Foo'), ["Foo"])
 262         assert_equal(db.metadata_keys('A'), [])
 263         assert_equal(db.metadata_keys('Food'), [])
 264         assert_equal(db.metadata_keys('f'), [])
 265         count = 0
 266         db.metadata_keys { |k|
 267           count += 1
 268           assert_equal(k, "Foo")
 269         }
 270         assert_equal(1, count)
 271     }
 272   end
 273
 274   def test_013_scaleweight
 275     query = Xapian::Query.new("foo")
 276     query2 = Xapian::Query.new(Xapian::Query::OP_SCALE_WEIGHT, query, 5)
 277     assert_equal(query2.description(), "Query(5 * foo)")
 278   end
 279
 280   def test_014_sortable_serialise
 281     # In Xapian 1.0.13/1.1.1 and earlier, the SWIG generated wrapper code
 282     # didn't handle integer values > MAXINT for double parameters.
 283     v = 51767811298
 284     assert_equal(v, Xapian::sortable_unserialise(Xapian::sortable_serialise(v)))
 285   end
 286
 287   def test_015_valuecount_matchspy
 288     spy = Xapian::ValueCountMatchSpy.new(0)
 289     doc = Xapian::Document.new()
 290     doc.add_posting("term", 1)
 291     doc.add_value(0, "yes")
 292     @db.add_document(doc)
 293     @db.add_document(doc)
 294     doc.add_value(0, "maybe")
 295     @db.add_document(doc)
 296     doc.add_value(0, "no")
 297     @db.add_document(doc)
 298     query = Xapian::Query.new("term")
 299     enquire = Xapian::Enquire.new(@db)
 300     enquire.query = query
 301     enquire.add_matchspy(spy)
 302     mset = enquire.mset(0, 10)
 303     assert_equal(mset.size(), 4)
 304     assert_equal(spy.values.map{|i| "%s:%d"%[i.term, i.termfreq]} * ",",
 305                  "maybe:1,no:1,yes:2")
 306     assert_equal(spy.top_values(1).map{|i| "%s:%d"%[i.term, i.termfreq]} * ",",
 307                  "yes:2")
 308     assert_equal(spy.top_values(2).map{|i| "%s:%d"%[i.term, i.termfreq]} * ",",
 309                  "yes:2,maybe:1")
 310     assert_equal(spy.top_values(3).map{|i| "%s:%d"%[i.term, i.termfreq]} * ",",
 311                  "yes:2,maybe:1,no:1")
 312
 313     # Test the valuestream iterator, while we've got some data
 314     assert_equal(@db.valuestream(1).size(), 0)
 315     assert_equal(@db.valuestream(0).map{|i| "%d:%s"%[i.docid, i.value]}*",",
 316                  "2:yes,3:yes,4:maybe,5:no")
 317   end
 318
 319   def test_016_compactor
 320     Dir.mktmpdir("smokerb") {|tmpdir|
 321         db1path = "#{tmpdir}/db1"
 322         db2path = "#{tmpdir}/db2"
 323         db3path = "#{tmpdir}/db3"
 324
 325         # Set up a couple of sample input databases
 326         db1 = Xapian::WritableDatabase.new(db1path, Xapian::DB_CREATE_OR_OVERWRITE)
 327         doc1 = Xapian::Document.new()
 328         doc1.add_term('Hello')
 329         doc1.add_term('Hello1')
 330         doc1.add_value(0, 'Val1')
 331         db1.set_metadata('key', '1')
 332         db1.set_metadata('key1', '1')
 333         db1.add_document(doc1)
 334         db1.commit()
 335
 336         db2 = Xapian::WritableDatabase.new(db2path, Xapian::DB_CREATE_OR_OVERWRITE)
 337         doc2 = Xapian::Document.new()
 338         doc2.add_term('Hello')
 339         doc2.add_term('Hello2')
 340         doc2.add_value(0, 'Val2')
 341         db2.set_metadata('key', '2')
 342         db2.set_metadata('key2', '2')
 343         db2.add_document(doc2)
 344         db2.commit()
 345
 346         # Compact with the default compactor
 347         # Metadata conflicts are resolved by picking the first value
 348         db_to_compact = Xapian::Database.new()
 349         db_to_compact.add_database(Xapian::Database::new(db1path))
 350         db_to_compact.add_database(Xapian::Database::new(db2path))
 351         db_to_compact.compact(db3path)
 352
 353         db3 = Xapian::Database.new(db3path)
 354         #assert_equal([(item.term, item.termfreq) for item in db3.allterms()],
 355         #       [('Hello', 2), ('Hello1', 1), ('Hello2', 1)])
 356         assert_equal(db3.document(1).value(0), 'Val1')
 357         assert_equal(db3.document(2).value(0), 'Val2')
 358         assert_equal(db3.get_metadata('key'), '1')
 359         assert_equal(db3.get_metadata('key1'), '1')
 360         assert_equal(db3.get_metadata('key2'), '2')
 361     }
 362   end
 363
 364   def test_017_latlongcoords_iterator
 365     coords = Xapian::LatLongCoords.new()
 366     coords.append(Xapian::LatLongCoord.new(0, 0))
 367     assert_equal(coords.size(), 1)
 368     assert_equal(coords.all.map{|i| "%s"%i.description}*",",
 369                  "Xapian::LatLongCoord(0, 0)")
 370     s = ''
 371     coords.all {|i| s += i.description }
 372     assert_equal(s, "Xapian::LatLongCoord(0, 0)")
 373   end
 374
 375   def test_018_spellings
 376     # The inmemory backend doesn't support spellings so we need to create a
 377     # "real" database for these tests.
 378     Dir.mktmpdir("smokerb") {|tmpdir|
 379         dbpath = "#{tmpdir}/dbspell"
 380
 381         db = Xapian::WritableDatabase.new(dbpath, Xapian::DB_CREATE_OR_OVERWRITE)
 382         a = []
 383         db.spellings { |x| a.push(x) }
 384         assert_equal(a, db.spellings)
 385         assert_equal([], a)
 386         db.add_spelling("there")
 387         db.add_spelling("their")
 388         db.add_spelling("they're")
 389         db.add_spelling("there")
 390         a = []
 391         db.spellings { |x| a.push(x) }
 392         assert_equal(a, db.spellings)
 393         assert_equal([Xapian::Term.new("their", 0, 1),
 394                       Xapian::Term.new("there", 0, 2),
 395                       Xapian::Term.new("they're", 0, 1)], a)
 396     }
 397   end
 398
 399   def test_019_synonyms
 400     # The inmemory backend doesn't support synonyms so we need to create a
 401     # "real" database for these tests.
 402     Dir.mktmpdir("smokerb") {|tmpdir|
 403         dbpath = "#{tmpdir}/dbsynonym"
 404
 405         db = Xapian::WritableDatabase.new(dbpath, Xapian::DB_CREATE_OR_OVERWRITE)
 406
 407         k = []
 408         db.synonym_keys { |x| k.push(x) }
 409         assert_equal(k, db.synonym_keys)
 410         assert_equal(k, [])
 411
 412         a = []
 413         db.synonyms('food') { |x| a.push(x) }
 414         assert_equal(a, db.synonyms('food'))
 415         assert_equal(a, [])
 416
 417         db.add_synonym("food", "nosh")
 418         db.add_synonym("food", "grub")
 419         db.add_synonym("food", "kai")
 420         db.add_synonym("drink", "tea")
 421         db.add_synonym("drink", "coffee")
 422
 423         k = []
 424         db.synonym_keys { |x| k.push(x) }
 425         assert_equal(k, db.synonym_keys)
 426         assert_equal(['drink', 'food'], k)
 427
 428         a = []
 429         db.synonyms('drink') { |x| a.push(x) }
 430         assert_equal(a, db.synonyms('drink'))
 431         assert_equal(['coffee', 'tea'], a)
 432
 433         a = []
 434         db.synonyms('food') { |x| a.push(x) }
 435         assert_equal(a, db.synonyms('food'))
 436         assert_equal(['grub', 'kai', 'nosh'], a)
 437
 438         a = []
 439         db.synonyms('nothing') { |x| a.push(x) }
 440         assert_equal(a, db.synonyms('nothing'))
 441         assert_equal([], a)
 442     }
 443   end
 444
 445   def test_020_queryparser
 446     stopper = Xapian::SimpleStopper.new()
 447     stopper.add('a')
 448     stopper.add('the')
 449     qp = Xapian::QueryParser.new()
 450     qp.stopper = stopper
 451     qp.stemmer = Xapian::Stem.new('en')
 452     q = qp.parse_query("The starting started with a start")
 453     puts q.description
 454
 455     a = []
 456     qp.stoplist() { |x| a.push(x) }
 457     assert_equal(a, qp.stoplist())
 458     assert_equal(['the', 'a'], a)
 459
 460     a = []
 461     qp.unstem('Zthe') { |x| a.push(x) }
 462     assert_equal(a, qp.unstem('Zthe'))
 463     assert_equal([], a)
 464
 465     a = []
 466     qp.unstem('Zstart') { |x| a.push(x) }
 467     assert_equal(a, qp.unstem('Zstart'))
 468     assert_equal(['starting', 'started', 'start'], a)
 469   end
 470
 471 end # class XapianSmoketest