xapian-applications/omega/utf8truncate.cc

   1 /** @file
   2  * @brief truncate a utf-8 string, ideally without splitting words.
   3  */
   4 /* Copyright (C) 2007,2021 Olly Betts
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
  19  */
  20
  21 #include <config.h>
  22
  23 #include "utf8truncate.h"
  24 #include <string>
  25
  26 using namespace std;
  27
  28 bool
  29 utf8_truncate(string& value, string::size_type maxlen)
  30 {
  31     if (value.size() <= maxlen) return false;
  32
  33     string::size_type len = maxlen + 1;
  34     // Skip back to (and past) the last whitespace.  We start one past the
  35     // length we want to correctly handle the case where a word ends exactly
  36     // maxlen bytes in.
  37     while (len && static_cast<unsigned char>(value[len - 1]) > 32) --len;
  38     while (len && static_cast<unsigned char>(value[len - 1]) <= 32) --len;
  39
  40     // If the first word is too long, truncate it.
  41     if (!len) {
  42         len = maxlen;
  43         // If the bytes of a UTF-8 character span the maxlen position we need
  44         // to remove some extra bytes to avoid leaving a partial UTF-8
  45         // character.
  46         //
  47         // We start at the byte after the cut point.  If it's a continuation
  48         // byte we step back until we find the start of the character and
  49         // truncate right before that.
  50         while (len && (value[len] & 0xc0) == 0x80) --len;
  51     }
  52     value.resize(len);
  53
  54     return true;
  55 }