#' Workflow to patch CWB

#'

#' Flag -fcommon removed from config and config/platform/linux again

#' @examples

#' source("~/Lab/github/RcppCWB/patch/PatchCWB.R")

#'

#' P <- PatchCWB$new(

#' cwb_dir_svn = "~/Lab/tmp/cwb/trunk",

#' repodir = "~/Lab/github/RcppCWB",

#' makeheaders = "~/Lab/github_foreign/makeheaders/src/makeheaders",

#' revision = 1690

#' )

#' P$patch_all()

PatchCWB <- R6Class(

classname = "PatchCWB",

public = list(

#' @field global_replacements Generated `$initialize()` by calling

#' `$configure_global_replacements`

revision = NULL,

cwb_dir_svn = NULL,

repodir = NULL,

repository = NULL,

branch_of_departure = NULL,

last_commit = NULL,

patch_commit = NULL,

patchbranch = NULL,

global_replacements = NULL,

file_patches = NULL,

verbose = TRUE,

diff_global_replacements = NULL,

diff_file_patches = NULL,

makeheaders = NULL,

initialize = function(cwb_dir_svn, revision, repodir, makeheaders, verbose = TRUE){

self$verbose <- verbose

self$repodir <- path.expand(repodir)

if (self$verbose) message("RcppCWB repository path: ", self$repodir)

self$cwb_dir_svn <- path.expand(cwb_dir_svn)

if (self$verbose) message("SVN directory with CWB code: ", self$cwb_dir_svn)

self$revision <- as.integer(revision)

if (self$revision) message("SVN revision to use: ", self$revision)

self$branch_of_departure <- self$get_branch_of_departure()

if (self$verbose) message("The branch of departure is: ", self$branch_of_departure)

self$repository <- repository(self$repodir)

self$last_commit <- last_commit(repo = self$repodir)

self$global_replacements <- self$configure_global_replacements(revision)

self$file_patches <- self$patches(revision)

message("flex version: ", system("flex --version", intern = TRUE))

message("bison version: ", system("bison --version", intern = TRUE)[1])

message("makeheaders utility is available: ", identical(system(makeheaders, intern = TRUE), character()))

self$makeheaders <- makeheaders

invisible(self)

},

get_branch_of_departure = function(){

branches(self$repodir)[sapply(branches(self$repodir), is_head)][[1]][["name"]]

},

svn_get_revision = function(){

old_wd <- setwd(self$cwb_dir_svn)

rev <- as.integer(system("svn info --show-item revision", intern = TRUE))

setwd(old_wd)

rev

},

#' @return An `integer` value with the active SVN revision number.

svn_set_revision = function(){

old_wd <- setwd(self$cwb_dir_svn)

rev <- self$svn_get_revision()

if (self$verbose) message("Active revision of CWB SVN repository: ", rev)

if (rev != as.integer(self$revision)){

system(

sprintf("svn up -r %d", self$revision),

intern = TRUE

)

rev <- self$svn_get_revision()

if (self$verbose) message("Updated revision of CWB SVN repository: ", rev)

}

setwd(old_wd)

rev

},

cwb_fresh_copy = function(){

if (self$verbose) message("* copy unaltered CWB code into RcppCWB repository ... ", appendLF = FALSE)

self$patchbranch <- sprintf("r%d", self$svn_get_revision()) # name of new branch

git2r::checkout(self$repodir, branch = self$branch_of_departure)

if (self$patchbranch %in% names(branches(self$repodir))){

git2r::branch_delete(branch = branches(self$repodir)[[self$patchbranch]])

}

git2r::branch_create(commit = last_commit(repo = self$repodir), name = self$patchbranch)

git2r::checkout(self$repodir, branch = self$patchbranch) # switch to new branch

# copy CWB files

repo_cwb_dir <- path(self$repodir, "src", "cwb")

cwb_files <- list.files(self$cwb_dir_svn, recursive = TRUE, full.names = TRUE)

file.copy(

from = cwb_files,

to = gsub(paste("^", self$cwb_dir_svn, sep = ""), repo_cwb_dir, cwb_files),

overwrite = TRUE

)

if (self$verbose) message(sprintf("%d files copied", length(cwb_files)))

# Remove files that have been added (and that need to be added explicitly)

# To restore the state of RcppCWB development, these files need to be re-added or generated later on

cwb_files_svn_truncated <- gsub(self$cwb_dir_svn, "", cwb_files)

cwb_files_repo <- list.files(repo_cwb_dir, recursive = TRUE, full.names = TRUE)

cwb_files_repo_truncated <- gsub(repo_cwb_dir, "", cwb_files_repo)

# config dir copy of platform-file 'solaris': 'solaris_x86', with modification 'CFLAGS = -Wall -O3 -fPIC'

added_files <- cwb_files_repo_truncated[!cwb_files_repo_truncated %in% cwb_files_svn_truncated]

added_files_full_path <- fs::path(repo_cwb_dir, added_files)

added_files_existing <- added_files_full_path[file.exists(added_files_full_path)]

git2r::rm_file(repo = self$repodir, path = added_files_existing)

# add & commit

git2r::add(repo = self$repodir, path = unname(unlist(git2r::status(repo = self$repodir))))

git2r::commit(self$repodir, message = "CWB restored")

},

#' @description

#' The flex and bison parsers may be missing on CRAN build machines. To

#' avoid errors, parsed files files are generated (registry.tab.c,

#' registry.tab.h, registry.y)

run_bison_flex = function(){

if (self$verbose) message("* run bison and flex parsers")

cwb_pkg_dir <- file.path(self$repodir, "src", "cwb")

old_wd <- setwd(path(cwb_pkg_dir, "cl"))

system("bison -d -t -p creg registry.y")

system("flex -8 -Pcreg registry.l")

setwd(path(cwb_pkg_dir, "cqp"))

system("bison -d -t parser.y")

system("flex -8 parser.l")

setwd(old_wd)

invisible(self)

},

rename_files = function(){

if (self$verbose) message("* rename files cl/endian.h (to endian2.h) and instutils/Makefile (to _Makefile)")

cwb_pkg_dir <- file.path(self$repodir, "src", "cwb")

file.rename(

from = path(cwb_pkg_dir, "cl", "endian.h"),

to = path(cwb_pkg_dir, "cl", "endian2.h")

)

file.rename(

from = path(cwb_pkg_dir, "instutils", "Makefile"),

to = path(cwb_pkg_dir, "instutils", "_Makefile")

)

invisible(self)

},

copy_files = function(){

if (self$verbose) message("* add newly created files to CWB code (overwriting existing files)")

new_files <- list.files(path = file.path(self$repodir, "patch", "cwb"), full.names = TRUE, recursive = TRUE)

for (fname in new_files){

if (self$verbose) message("... copy file: ", fname)

file.copy(from = fname, to = gsub("/patch/", "/src/", fname), overwrite = TRUE)

}

invisible(self)

},

create_copy = function(){

if (self$verbose) message ("* create utils/globals.h as a copy of cwb-encode.c")

file.copy(

from = file.path(self$repodir, "src", "cwb", "utils", "cwb-encode.c"),

to = file.path(self$repodir, "src", "cwb", "utils", "globals.h"),

overwrite = TRUE

)

file.copy(

from = file.path(self$repodir, "src", "cwb", "cqp", "eval.h"),

to = file.path(self$repodir, "src", "_eval.h"),

overwrite = TRUE

)

},

create_globalvars_file = function(){

if (self$verbose) message("* create _globalvars.h ... ", appendLF = FALSE)

# Strategy 1: Variables that are declared as 'extern' by patch are global vars

extern_by_patch <- unique(unname(unlist(lapply(self$file_patches, `[[`, "extern"))))

# Strategy 2: Later version of CWB prepends extern by default. Use makeheaders

# utility to autogenerate a header and grep externed vars

extern_by_default <- grep("^\\s*extern\\s", system(

sprintf("%s -h %s", self$makeheaders, file.path(self$repodir, "src", "cwb", "cqp", "options.c")),

intern = TRUE

), value = TRUE)

# combine results

extern <- unique(

c(sprintf("extern %s", extern_by_patch), extern_by_default)

)

# manual additions

typedef_CQPOption <- self$get_snippet(

file = file.path(self$repodir, "src", "cwb", "cqp", "options.h"),

from = "^\\s*typedef\\senum\\s_opttype\\s\\{\\s*$",

to = "^\\s*}\\s+CQPOption;\\s*$"

)

defs <- self$get_snippet(

file = file.path(self$repodir, "src", "cwb", "cqp", "options.h"),

from = "^#define\\sOPTION_VISIBLE_IN_CQP\\s+\\d+\\s*$",

to = '#define\\sCQP_FALLBACK_PAGER\\s"more"'

)

# options <- self$get_snippet(

# file = file.path(self$repodir, "src", "cwb", "cqp", "options.c"),

# from = "^CQPOption\\scqpoptions\\[\\]\\s=\\s\\{\\s*$",

# to = '\\s*\\*\\sNON-OPTION\\sGLOBAL\\sVARIABLES\\s*$'

# )

# options <- options[1:(length(options) - 3)]

extern <- c(

if (self$revision < 1690) "enum _which_app { undef, cqp, cqpcl, cqpserver} which_app;",

if (self$revision >= 1690) "typedef enum which_app { undef, cqp, cqpcl, cqpserver} which_app_t;",

typedef_CQPOption,

extern,

defs,

"extern CQPOption cqpoptions;"

)

extern <- extern[!extern %in% c("EvalEnvironment Environment[MAXENVIRONMENT];", "EEP CurEnv, evalenv;", "int eep;", "extern CQPOption cqpoptions[];")]

writeLines(text = extern, con = file.path(self$repodir, "src", "_globalvars.h"))

message("OK")

},

create_dummy_depend_files = function(){

if (self$verbose) message("* create dummy depend.mk files ...")

cat("\n", file = file.path(self$repodir, "src", "cwb", "cl", "depend.mk"))

cat("\n", file = file.path(self$repodir, "src", "cwb", "cqp", "depend.mk"))

invisible(self)

},

replace_globally = function(){

if (self$verbose) message("* perform global replacements ...")

cwb_pkg_dir <- file.path(self$repodir, "src", "cwb")

files <- c(

unlist(lapply(

c("cl", "cqp", "CQi"),

function(subdir) list.files(file.path(cwb_pkg_dir, subdir), full.names = TRUE)

)),

path.expand(

file.path(cwb_pkg_dir, "utils", c("cwb-encode.c", "cwb-makeall.c", "cwb-huffcode.c", "cwb-compress-rdx.c"))

)

)

for (f in files){

code <- readLines(f)

for (i in 1L:length(self$global_replacements)){

if (length(self$global_replacements[[i]]) > 2L){

if (endsWith(f, self$global_replacements[[i]][[3]])) next

}

code <- gsub(self$global_replacements[[i]][1], self$global_replacements[[i]][2], code)

}

writeLines(text = code, con = f)

}

},

#' @param file A file

get_snippet = function(file, from, to){

code <- readLines(file)

start <- grep(from, code)

if (!length(start)) warning("Fn get_snippet - no match for query to get start: ", from)

end <- grep(to, code)

if (!length(end)) warning("Fn get_snippet - no match for query to get end: ", end)

code[start:end]

},

delete_line_before = function(code, action, file){

which_position <- if (length(action) > 1L) action[[2]] else 1L

times <- if (length(action) == 3L) action[[3]] else 1L

position <- grep(pattern = action[[1]], code)[which_position]

if (is.na(times)) times <- position - 1L

if (!is.na(position)){

code <- code[-(position - 1L:times)]

} else {

message(

sprintf("Trying to patch file '%s' - no match for action 'delete_line_before' (regex: %s | match: %d | lines: %d) ", file, action[[1]], which_position, times)

)

}

code

},

delete_line_beginning_with = function(code, action, file){

which_position <- if (length(action) > 1L) action[[2]] else 1L

times <- if (length(action) == 3L) action[[3]] else 1L

position <- grep(pattern = action[[1]], code)[which_position]

if (is.na(times)) times <- length(code) - position

if (!is.na(position)){

code <- code[-(position + 0L:times)]

} else {

message(

sprintf("Trying to patch file '%s' - no match for action 'delete_line_after' (regex: %s | match: %d | lines: %d) ", file, action[[1]], which_position, times)

)

}

code

},

insert_before = function(code, action, file){

which_position <- if (length(action) > 2L) action[[3]] else 1

position <- grep(pattern = action[[1]], code)[which_position]

if (!is.na(position)){

code <- c(

code[1L:(position - 1L)],

action[[2]],

code[position:length(code)]

)

} else {

message(

sprintf(

"Trying to patch file '%s' - no match for action 'insert_before' (regex: %s | match: %d | insertion: %s)",

file, action[[1]], which_position, paste(action[[2]], collapse = "///")

)

)

}

code

},

insert_after = function(code, action, file){

which_position <- if (length(action) > 2L) action[[3]] else 1L

position <- grep(pattern = action[[1]], code)[which_position]

if (!is.na(position)){

code <- c(

code[1L:position],

action[[2]],

code[(position + 1L):length(code)]

)

} else {

message(

sprintf(

"Trying to patch file '%s' - no match for action 'insert_after' (regex: %s | match: %d | insertion: %s)",

file, action[[1]], which_position, paste(action[[2]], collapse = "///")

)

)

}

code

},

replace = function(code, action, file){

position <- grep(pattern = action[[1]], code)

if (length(position) == 0L){

message(

sprintf("Trying to patch file '%s' - no match for action 'replace' (regex: %s | match: %d | replacement: %s)",

file, action[[1]], action[[3]], paste(action[[2]], collapse = "///")),

appendLF = FALSE

)

return(code)

}

if (!is.na(action[[3]])) position <- position[ action[[3]] ]

for (p in position){

code[position] <- gsub(action[1], action[2], code[position])

}

code

},

remove_lines = function(code, action, file){

position <- grep(pattern = action[[1]], code)

position <- if (!is.na(action[[2]])) position[ action[[2]] ] else position

if (!is.na(position)){

code <- code[-position]

} else {

message(

sprintf("Trying to patch file '%s' - no match for action 'remove_lines' (regex: %s | match: %d)", file, action[[1]], action[[2]]),

appendLF = FALSE

)

}

code

},

extern = function(code, action, file){

if (length(action) == 0L) return(code)

for (ext in action){

matches <- which(startsWith(code, ext))

if (length(matches) > 0L){

for (position in matches){

code[position] <- paste("extern", code[position], sep = " ")

}

} else {

message(sprintf("Trying to patch file '%s' - no match for action 'extern' (var to extern: %s | n_matches: %d", file, ext, length(matches)))

}

}

code

},

configure_global_replacements = function(revision){

replacements <- list(

# In revision 1690, there are further targets dst->stream, outfh, tmp, fh

if (revision == 1069){

c("(vf|f|v)printf\\s*\\(\\s*(stderr|stream|stdout|outfd|File|rd->stream|redir->stream|debug_output|protocol),\\s*", "Rprintf(")

} else if (revision >= 1690){

c("(vf|f|v)printf\\s*\\(\\s*(stderr|stream|stdout|outfd|fd|File|rd->stream|redir->stream|debug_output|outfh|tmp|fh|dest|protocol|tmp_dst),\\s*", "Rprintf(")

},

if (revision == 1069) c("(vf|f|v)printf\\s*\\(\\s*fd,\\s*", "Rprintf(", "cwb-encode.c"),

c("YY(F|D)PRINTF\\s*(\\({1,2})\\s*(stderr|yyoutput),\\s*" , "YY\\1PRINTF \\2"),

# c("fprintf\\s*\\(", "Rprintf("),

c("(\\s+)printf\\(", "\\1Rprintf("),

c("#(\\s*)define\\sYYFPRINTF\\sfprintf", "#\\1define YYFPRINTF Rprintf"),

# The CWB file 'endian.h' causes issues with the endian.h system file. The

# best solution I could come up with is to rename endian.h into endian2.h.

# In addition - turn 'endian.h' into 'endian2.h' in the Makefile - change

# include statements to 'include "endian2.h"'

c('^\\s*#include\\s+"endian\\.h"\\s*$', '#include "endian2.h"') # only files in cl, maybe limit this,

# c("^(\\s*)exit\\(1\\);", "\\1return 1;")

)

for (i in which(sapply(replacements, is.null) == TRUE)) replacements[[i]] <- NULL

replacements

},

# Order to maintain:

# - delete_line_before

# - insert_before

# - insert_after

# - replace

# - remove_lines

patches = function(revision){

list(

"src/cwb/cl/lex.creg.c" = c(

list(),

# These changes are highly specific and are unlikely to work out of the box for r1690

if (revision == 1069) list(

delete_line_before = list("^\\s*if\\s\\(\\s\\(yy_c_buf_p\\)", 4L, 4L),

delete_line_before = list("^\\s*cregrestart\\(cregin\\s*\\);", 2L, 11L),

delete_line_before = list("/\\*\\*\\sImmediately\\sswitch\\sto\\sa\\sdifferent\\sinput\\sstream\\.", 1L, 1L),

insert_before = list("/\\*\\send\\sstandard\\sC\\sheaders\\.\\s\\*/", c("void Rprintf(const char *, ...);", ""), 1),

insert_before = list("#ifndef\\sYY_NO_INPUT", "/*", 1L),

insert_before = list("^\\s*static\\svoid\\syyunput\\s\\(int\\sc,\\sregister\\schar\\s\\*\\syy_bp\\s\\)", "/*", 1L),

insert_before = list("^#ifndef\\sYY_NO_INPUT", "/*", 2L),

insert_before = list("/\\*\\*\\sImmediately\\sswitch\\sto\\sa\\sdifferent\\sinput\\sstream\\.", c("*/", ""), 1L),

insert_after = list("#endif", "*/", 28L),

insert_after = list("^\\}$", "*/", 5L),

# Funktion 'static void yyunput (int c, register char * yy_bp )'

# auskommentiert, zur Vermeidung Nachricht: lex.creg.c:1410:17:

# warning: unused function 'yyunput' [-Wunused-function]

replace = list("^(\\s*)(static\\svoid\\syyunput\\s\\(int\\sc,char\\s\\*buf_ptr\\s+\\);).*?$", "\\1/* \\2 */", 1),

replace = list("^\\s*static\\svoid\\syyunput\\s\\(int\\sc,\\sregister\\schar\\s\\*\\syy_bp\\s\\)", "static void yyunput (int c, register char * yy_bp )", 1L),

replace = list("^(\\s*)\\{(\\s*)/\\*\\sneed\\smore\\sinput\\s\\*/", "\\1{\\2", 1L),

replace = list("/\\*\\scast\\sfor\\s8-bit\\schar's\\s\\*/", "", 1L),

replace = list("\\s*/\\*\\spreserve\\scregtext\\s\\*/", "", 1L),

replace = list("/\\*\\sifndef YY_NO_INPUT\\s\\*/", "", 1L),

remove_lines = list("^\\s/\\*\\sundo\\seffects\\sof\\ssetting\\sup\\scregtext\\s\\*/", 1L),

remove_lines = list("/\\*\\sneed\\sto\\sshift\\sthings\\sup\\sto\\smake\\sroom\\s\\*/", 1L),

remove_lines = list("/\\*\\s\\+2\\sfor\\sEOB\\schars\\.\\s\\*/", 1L),

remove_lines = list("^\\s*/\\*\\sThis\\swas\\sreally\\sa\\sNUL\\.\\s\\*/", 1L),

remove_lines = list("^\\s*/\\*FALLTHROUGH\\*/\\s*$", 1L)

)

),

"src/cwb/cl/endian2.h" = c(

list(

insert_before = list("#include\\s<windows\\.h>" , "#include <winsock2.h> /* AB reversed order, in original CWB code windows.h is included first */", 1L)

),

# Ensure that order of inclusion is 'winsock2.h' and then 'windows.h'.

# The windows.h/winsock2.h order has persisted, but the comment after

# has changed.

if (revision == 1069) list(

delete_line_before = list("/* for consistency:", 1L, 1L)

),

if (revision >= 1690) list(

delete_line_before = list("/\\*\\snote,\\sin\\sorder\\sfor\\sall\\sthis\\sto\\swork,", 1L, 1L)

)

),

"src/cwb/cl/Makefile" = list(

delete_line_before = list("^libcl.a: \\$\\(OBJS\\)", 1L, if (revision >= 1400) 9L else 6L),

delete_line_before = list("^depend:$", 1L, if (revision >= 1400) 14L else 22L),

replace = list("^TOP\\s=\\s\\$\\(shell\\spwd\\)/\\.\\.", "TOP = $(R_PACKAGE_SOURCE)", 1L),

replace = list("^(\\s+)endian.h", "\\1endian2.h", 1L),

replace = list("^(\\s+)\\$\\(AR\\)\\s", "\\1$(AR) cq ", 1L),

remove_lines = list("^\\s+\\$\\(RANLIB\\)", 1L),

remove_lines = list('^\\s*@\\$\\(ECHO\\)\\s".*?"\\s*$', NA) # r1690 is beautiful - but we nee verbosity

),

"src/cwb/cl/attributes.c" = c(

list(

# This will work for r1690 too

insert_before = list("^#include\\s<ctype\\.h>", c("void Rprintf(const char *, ...);", "")),

insert_after = list(

'^#include\\s+"makecomps\\.h".*?$',

c(

"",

"#ifdef __MINGW__",

"#undef SUBDIR_SEPARATOR",

"#undef SUBDIR_SEP_STRING",

"#define SUBDIR_SEPARATOR '/'",

'#define SUBDIR_SEP_STRING "/"',

"#endif",

""

),

1L

)

),

# Cannot find the dollar variabl in r1690 - seems to be gone

if (revision == 1069) list(

# attributes.c:755:19: warning: variable ???dollar??? set but not used [-Wunused-but-set-variable]

# int ppos, bpos, dollar, rpos;

replace = list("(\\s+)int\\sppos,\\sbpos,\\sdollar,\\srpos;", "\\1int ppos, bpos, rpos;", 1),

replace = list("^(\\s+)dollar = 0;", "\\1/* dollar = 0; */", 1),

replace = list("^(\\s+)dollar = ppos;\\s", "\\1/* dollar = ppos; */", 1),

# The STREQ macro is replaced by a cl_str_is() function in r1690

# attributes.c: In function ???component_full_name???:

# macros.h:59:22: warning: the address of ???rname??? will always evaluate as ???true??? [-Waddress]

# ((a) && (b) && (strcmp((a), (b)) == 0)))

# attributes.c:807:11: note: in expansion of macro ???STREQ???

# if (STREQ(rname, "HOME"))

replace = list('^(\\s+)if\\s\\(STREQ\\(rname,\\s"HOME"\\)\\)', '\\1if (strcmp(rname, "HOME") == 0)', 1),

# attributes.c:809:16: note: in expansion of macro ???STREQ???

# else if (STREQ(rname, "APATH"))

replace = list('^(\\s+)else\\sif\\s\\(STREQ\\(rname,\\s"APATH"\\)\\)', '\\1else if (strcmp(rname, "APATH") == 0)', 1),

# attributes.c:812:16: note: in expansion of macro ???STREQ???

# STREQ macro dissolved to avoid warnings in attributes.c

# else if (STREQ(rname, "ANAME"))

replace = list('^(\\s+)else\\sif\\s\\(STREQ\\(rname,\\s"ANAME"\\)\\)', '\\1else if (strcmp(rname, "ANAME") == 0)', 1)

)

),

"src/cwb/cl/bitio.c" = list(

# Unchanged, will work as in 1069

insert_before = list("^#include\\s<sys/types\\.h>", "void Rprintf(const char *, ...);")

),

"src/cwb/cl/class-mapping.c" = c(

list(),

# This file does not exist in r1960

if (revision == 1069) list(

insert_before = list('#include\\s"globals\\.h"', "void Rprintf(const char *, ...);"),

# argumet names class renamed to obj

replace = list("(\\s*)\\*\\s@param\\sclass\\s+The\\sclass\\s+to\\scheck\\.", "\\1* @param obj The class to check.", 2),

replace = list("(\\s*)\\*\\s@param\\sclass\\s+The\\sclass\\s+to\\scheck\\.", "\\1* @param obj The class to check.", 1),

replace = list("^(\\s+)return\\smember_of_class_i\\(map,\\sclass,\\sid\\);", "\\1return member_of_class_i(map, obj, id);", 1),

replace = list("^(\\s+)SingleMapping\\sclass,", "\\1SingleMapping obj,", 2),

replace = list("^(\\s+)SingleMapping\\sclass,", "\\1SingleMapping obj,", 1),

replace = list("^(\\s+)class->tokens,", "\\1obj->tokens,", 1),

replace = list("^(\\s+)class->nr_tokens,", "\\1obj->nr_tokens,", 1)

)

),

"src/cwb/cl/corpus.c" = c(

list(

# the function and all uses of the function are not defined for MINGW - should be true for header, too

insert_before = list('#include\\s<ctype\\.h>', "void Rprintf(const char *, ...);"),

# corpus.c:37:12: warning: 'memberIDList' declared 'static' but never defined [-Wunused-function]

# 37 | static int memberIDList(char *s, IDList l);

# | ^~~~~~~~~~~~

insert_before = list("^\\s*static\\sint\\smemberIDList\\(char\\s\\*s,\\sIDList\\sl\\);\\s*$", "#ifndef __MINGW__", 1L),

insert_after = list("^\\s*static\\sint\\smemberIDList\\(char\\s\\*s,\\sIDList\\sl\\);\\s*$", "#endif", 1L)

),

# Usage of stderr on a separate line - 3 times in r1069, but only once in r1690

if (revision == 1069) list(

remove_lines = list("(\\s+)stderr,", 3),

remove_lines = list("(\\s+)stderr,", 2)

),

list(

# remove_lines = list("(\\s+)stderr,", 1),

delete_line_before = list("^\\s*memberIDList\\(char\\s\\*s,\\sIDList\\sl\\)\\s*$", 1L, 1L),

insert_before = list("^\\s*memberIDList\\(char\\s\\*s,\\sIDList\\sl\\)\\s*$", c("#ifndef __MINGW__", "static int"), 1L),

insert_before = list("^\\s*/\\*\\s-+\\s\\*/\\s*$", "#endif", 7L)

)

),

"src/cwb/cl/fileutils.c" = list(

# All if this should work in r1690 too (though patches for Solares may be obsolete)

insert_before = list('^#include\\s<sys/stat\\.h>', "void Rprintf(const char *, ...);"),

# to satisfy solaris

# #include <signal.h> /* added by Andreas Blaette */

# #include <sys/socket.h> /* added by Andreas Blaette */

insert_before = list('^#include\\s"globals\\.h"', c("#ifndef __MINGW__", "#include <signal.h> /* added by Andreas Blaette */", "#include <sys/socket.h> /* added by Andreas Blaette */", "#endif", "")),

# fileutils.c:400:19: warning: variable 'was_pipe' set but not used [-Wunused-but-set-variable]

insert_before = list(

"^(\\s*)int\\sresult\\s=\\s0,\\swas_pipe\\s=\\s0;\\s*$",

c("#ifndef __MINGW__", "int was_pipe;", "#endif"),

1L

),

replace = list("^(\\s*)int\\sresult\\s=\\s0,\\swas_pipe\\s=\\s0;\\s*$", "\\1int result = 0;", 1L),

insert_before = list("^\\s*was_pipe = 1;\\s*$", "#ifndef __MINGW__", 1L),

insert_after = list("^\\s*was_pipe = 1;\\s*$", "#endif", 1L),

# fileutils.c:188:1: warning: 'cl_handle_sigpipe' defined but not used [-Wunused-function]

delete_line_before = list("^\\s*cl_handle_sigpipe\\(int signum\\)\\s*$", 1L, 1L),

insert_before = list("^\\s*cl_handle_sigpipe\\(int signum\\)\\s*$", c("#ifndef __MINGW__", "static void"), 1L),

insert_before = list("^\\s*/\\*\\*\\scheck\\swhether\\sstream\\stype\\sinvolves\\sa\\spipe\\s\\*/\\s*$", "#endif", 1L)

),

"src/cwb/cl/globals.h" = list(

# Note the change from _globals_h_ to _cl_globals_h_

insert_before = list(

if (revision == 1069) '^#ifndef\\s_globals_h_' else '^#ifndef\\s_cl_globals_h_',

"void Rprintf(const char *, ...);")

),

"src/cwb/cl/lexhash.c" = list(

insert_before = list('#include\\s"globals\\.h"', "void Rprintf(const char *, ...);"),

# Do not recall why this include is necessary, but the lexhash.h include (and file)

# are absent in r1690 - so use another anchor.

insert_after = list(

if (revision == 1069) '^#include\\s"lexhash\\.h"' else '#include\\s"globals\\.h"',

"#include <unistd.h>"

)

),

"src/cwb/cl/macros.c" = list(

# Unchanged in r1690

insert_before = list('#include\\s"globals\\.h"', "void Rprintf(const char *, ...);")

),

"src/cwb/cl/makecomps.c" = c(

list(

# Unchanged in r1690

insert_before = list('#include\\s<ctype\\.h>', c("void Rprintf(const char *, ...);", ""))

),

if (revision == 1069) list(

# commented out: char errmsg[CL_MAX_LINE_LENGTH] because also defined elsewhere

replace = list("^(\\s*)char\\serrmsg\\[CL_MAX_LINE_LENGTH\\];", "/* char errmsg[CL_MAX_LINE_LENGTH]; */", 1)

)

),

"src/cwb/cl/registry.y" = list(

# Stable r1069-1690

insert_before = list('#include\\s<ctype\\.h>', "void Rprintf(const char *, ...);")

),

"src/cwb/cl/special-chars.c" = list(

# Stable r1069-1690

insert_before = list('#include\\s<ctype\\.h>', "void Rprintf(const char *, ...);")

),

"src/cwb/cl/storage.c" = list(

# All of this is stable r1069-1690

insert_before = list('^#include\\s<sys/types\\.h>', "void Rprintf(const char *, ...);"),

delete_line_beginning_with = list('#include "endian2.h"', 1L, 0L),

replace = list("^(\\s*)exit\\(1\\);", "\\1return;", NA),

insert_after = list('^#include\\s<sys/types\\.h>', '#include "endian2.h"'),

# storage.c: In function ???mmapfile???:

# storage.c:335:12: warning: ignoring return value of ???write???, declared with attribute warn_unused_result [-Wunused-result]

# write(fd, &fd, sizeof(int));

insert_before = list("^(\\s*)lseek\\(fd,\\s0,\\sSEEK_SET\\);", ' if (success < 0) Rprintf("Operation not successful");'),

replace = list("^(\\s+)write\\(fd,\\s&fd,\\ssizeof\\(int\\)\\);", "\\1ssize_t success = write(fd, &fd, sizeof(int));", 1)

),

"src/cwb/cl/windows-mmap.c" = list(

# stable r1069 and r1690

insert_before = list('^#include\\s"windows-mmap\\.h"', "void Rprintf(const char *, ...);")

),

# This file is not present in r1690

"src/cwb/cl/registry.tab.c" = list(

insert_before = list("#define\\sYYBISON\\s1", "void Rprintf(const char *, ...);", 1L)

),

"src/cwb/cl/bitfields.c" = list(

# stable r1069-r1690

insert_after = list("^static\\sint\\sBaseTypeBits", "void Rprintf(const char *, ...);")

),

"src/cwb/cl/cdaccess.c" = c(

list(

# The include of 'cdaccess.h' is gone with r1690 so we use another anchor

insert_after = list(

if (revision == 1069) '^#include\\s"cdaccess\\.h"' else '^#include\\s+"compression\\.h"',

c(

"#include <stdint.h>",

"void Rprintf(const char *, ...);"

)

),

# This code is unchanged with r1690

# cdaccess.c:2697:12: warning: ignoring return value of ???fgets???, declared with attribute warn_unused_result [-Wunused-result]

# fgets(call, CL_MAX_LINE_LENGTH, pipe);

replace = list("^(\\s*)fgets\\(call,\\sCL_MAX_LINE_LENGTH,\\spipe\\);", '\\1if (fgets(call, CL_MAX_LINE_LENGTH, pipe) == NULL) Rprintf("fgets failure");', 1),

# cdaccess.c: In function 'cl_read_stream':

# cdaccess.c:982:5: warning: 'memcpy' specified bound between 18446744065119617024 and 18446744073709551612 exceeds maximum object size 9223372036854775807 [-Wstringop-overflow=]

# 982 | memcpy(buffer, ps->base + ps->nr_items, items_to_read * sizeof(int));

insert_before = list(

"^\\s*memcpy\\(buffer,\\sps->base\\s\\+\\sps->nr_items,\\sitems_to_read\\s\\*\\ssizeof\\(int\\)\\);\\s*$",

c(

" size_t k;",

" k = items_to_read * sizeof(int);",

" if (k < PTRDIFF_MAX){"

)

),

replace = list(

"^\\s*memcpy\\(buffer,\\sps->base\\s\\+\\sps->nr_items,\\sitems_to_read\\s\\*\\ssizeof\\(int\\)\\);\\s*$",

" memcpy(buffer, ps->base + ps->nr_items, k);", 1L

),

insert_before = list("^(\\s*)return\\sitems_to_read;\\s*$", " }", 1L)

),

# These are patches to omit 'unused variable' warnings gone with r1690 (vars commented out, for instance)

if (revision == 1069) list(

# unchanged in r1690? But does not match

replace = list("^(\\s*)off_end\\s=\\sntohl\\(lexidx_data\\[idx\\s\\+\\s1\\]\\)\\s-\\s1;", "\\1/* off_end = ntohl(lexidx_data[idx + 1]) - 1; */", 1),

replace = list("^(\\s*)int\\sregex_result,\\sidx,\\si,\\slen,\\slexsize;", "\\1int idx, i, lexsize;", 1),

replace = list("^(\\s*)int\\soptimised,\\sgrain_match;", "\\1int optimised;", 1),

replace = list("^(\\s*)char\\s\\*word,\\s\\*preprocessed_string;", "\\1char *word;", 1),

# cdaccess.c: In function ???cl_regex2id???:

# cdaccess.c:1392:20: warning: variable ???off_end??? set but not used [-Wunused-but-set-variable]

# int off_start, off_end; /* start and end offset of current lexicon entry */

replace = list("^(\\s*)int\\soff_start,\\soff_end;", "\\1int off_start;", 1),

replace = list("^(\\s*)char\\s\\*p;", "\\1/* char *p; */", 1),

replace = list("^(\\s*)int\\si;", "\\1/* int i; */", 3),

# cdaccess.c: In function ???cl_dynamic_call???:

# cdaccess.c:2533:17: warning: variable ???arg??? set but not used [-Wunused-but-set-variable]

# DynCallResult arg;

replace = list("^(\\s*)DynCallResult\\sarg;", "\\1/* DynCallResult arg; */", 1),

replace = list("^(\\s*)arg\\s=\\sargs\\[argnum\\];", "\\1/* arg = args[argnum]; */", 1)

)

),

"src/cwb/cl/globals.c" = list(

# stable r10690 - r1690

insert_after = list(

'^#include\\s"globals\\.h"',

c(

"void Rprintf(const char *, ...);",

"",

"char* cl_get_version(){",

" #ifdef CWB_VERSION",

" char* version = CWB_VERSION;",

" #else",

' char* version = "";',

" #endif",

" return version;",

"}",

"",

"char* cl_get_p_attr_default(){",

" #ifdef CWB_DEFAULT_ATT_NAME",

" char* p_attr = CWB_DEFAULT_ATT_NAME;",

" #else",

' char* p_attr = "";',

" #endif",

" return p_attr;",

"}"

)

),

# This is propably the biggest intervention into the original CWB code

# Setting the locale to C is explained as follows:

# /* setting the locale to C makes the use of locale-sensitive Glib functions

# * behave as if they are locale insensitive; result is constant behaviour. */

# The price is just to high, because re-setting the locale breaks polmineR

# and violates CRAN rules.

replace = list('^\\s*(#include\\s<locale\\.h>)\\s*$', "/* \\1 */", 1L),

replace = list('^(\\s*)(setlocale\\(LC_ALL,\\s"C"\\);)\\s*$', "\\1/* \\2 */", 1L)

),

"src/cwb/cl/ngram-hash.c" = c(

list(

insert_after = list("^#include\\s<math\\.h>", "void Rprintf(const char *, ...);")

),

# The variable 'temp' is unused only in r1069?!

if (revision == 1069) list(

# ngram-hash.c: In function ???cl_delete_ngram_hash???:

# ngram-hash.c:146:30: warning: variable ???temp??? set but not used [-Wunused-but-set-variable]

# cl_ngram_hash_entry entry, temp;

replace = list("^(\\s*)cl_ngram_hash_entry\\sentry,\\stemp;", "\\1cl_ngram_hash_entry entry;", 1),

replace = list("^(\\s*)temp\\s=\\sentry;", "\\1/* temp = entry; */", 1)

)

),

"src/cwb/cl/regopt.c" = c(

list(

insert_after = list(

if (revision == 1069) '^#include\\s"regopt\\.h"' else '^#include\\s"globals\\.h"',

"void Rprintf(const char *, ...);"

)

),

# The issue may still be present but the syntax has changed - so we do not apply this for now

if (revision == 1069) list(

# regopt.c: In function ???make_jump_table???:

# regopt.c:1148:18: warning: suggest parentheses around comparison in operand of ???&??? [-Wparentheses]

# if (ch >= 32 & ch < 127)

replace = list("^(\\s*)if\\s\\(ch\\s>=\\s32\\s&\\sch\\s<\\s127\\)", "\\1if ((ch >= 32) & (ch < 127))", 1)

)

),

"src/cwb/cl/class-mapping.h" = c(

list(),

# The file is not there in r1690

if (revision == 1069) list(

replace = list("^(\\s+)SingleMapping\\sclass,", "\\1SingleMapping obj,", 2),

replace = list("^(\\s+)SingleMapping\\sclass,", "\\1SingleMapping obj,", 1)

)

),

# This file has appeared along the way, it just needs the minimal header

"src/cwb/cl/ui-helpers.c" = c(

list(),

if (revision >= 1690) list(

insert_before = list('^#include\\s"cl\\.h"', c("void Rprintf(const char *, ...);", ""), 1L)

)

),

"src/cwb/cl/cwb-globals.h" = c(

list(),

if (revision >= 1690) list(

delete_line_beginning_with = list("#if\\s__STDC_VERSION__\\s>=\\s199901L", 1L, 22L)

)

),

"src/cwb/cqp/groups.c" = c(

list(),

if (revision == 1069) list(

# Comment out open_temporary_file(char *tmp_name_buffer) in

# cqp/output.c, in cqp/output.h and in usage in functions calling

# this function: ComputeGroupExternally (cqp/groups.c) and

# SortExternally (cqp/ranges.c)

delete_line_before = list("^Group\\s\\*", 3L),

delete_line_before = list("^Group\\s\\*compute_grouping\\(CorpusList\\s\\*cl,", 1L),

delete_line_before = list("^Group\\s\\*compute_grouping\\(CorpusList\\s\\*cl,", 1L),

delete_line_before = list("^\\s*if\\s\\(\\(fd\\s=\\sopen_temporary_file\\(temporary_name\\)\\)\\s==\\sNULL\\)\\s\\{", 1L),

delete_line_before = list("^\\s*if\\s\\(\\(fd\\s=\\sopen_temporary_file\\(temporary_name\\)\\)\\s==\\sNULL\\)\\s\\{", 1L),

delete_line_before = list("^(\\s*)sprintf\\(sort_call,\\sExternalGroupingCommand,\\stemporary_name\\);", 1L),

delete_line_before = list("^\\s*return\\sComputeGroupExternally\\(group\\);", 1L),

insert_before = list("^Group\\s\\*", "/*", 3), # begin of commenting out ComputeGroupExternally(Group *group),

insert_before = list("^Group\\s\\*compute_grouping\\(CorpusList\\s\\*cl,", c("*/", "")), # end of commenting out ComputeGroupExternally(Group *group)

insert_after = list("return\\sComputeGroupInternally\\(group\\);", c(" */", " return group;")),

replace = list("^(.*?)\\s*/\\*\\s\\(source\\sID,\\starget\\sID)\\s*\\*/", "\\1", 1),

replace = list("^(.*?)\\s*/\\*\\smodifies\\sGroup\\sobject\\sin\\splace\\sand\\sreturns\\spointer\\s\\*/", "\\1", 1),

replace = list("^(\\s*)(if\\s\\(UseExternalGrouping\\s&&\\s\\!insecure\\s&&\\s\\!\\(source_is_struc\\s\\|\\|\\starget_is_struc\\s\\|\\|\\sis_grouped\\)\\))", "\\1/* \\2", 1)

),

if (revision == 1690) list(

delete_line_beginning_with = list("^\\s*if\\s\\(\\!\\(tmp_dst\\s=\\sopen_temporary_file\\(temporary_name\\)\\)\\)\\s\\{", 1L, 5L),

delete_line_beginning_with = list("^\\s*fclose\\(tmp_dst\\);\\s*$", 1L, 0L),

delete_line_beginning_with = list("^\\s*FILE\\s\\*tmp_dst;\\s*$", 1L, 0L),

# groups.c: In function 'get_group_id':

# groups.c:170:7: warning: 'pos' may be used uninitialized in this function [-Wmaybe-uninitialized]

# 170 | pos += offset; /* compute effective cpos */

# | ~~~~^~~~~~~~~

insert_after = list("^\\s*int\\spos,\\sid;\\s*$", " pos = -1;", 1L)

)

),

"src/cwb/cqp/output.c" = c(

list(

# output.c: In function 'pt_get_anchor_cpos':

# output.c:768:15: warning: 'cpos' may be used uninitialized in this function [-Wmaybe-uninitialized]

# 768 | return cpos + offset;

# | ~~~~~^~~~~~~~

insert_after = list("^\\s*int\\sreal_n,\\scpos;", " cpos = -1;", 1L)

),

if (revision < 1400) list(

# Comment out open_temporary_file(char *tmp_name_buffer) in

# cqp/output.c, in cqp/output.h and in usage in functions calling

# this function: ComputeGroupExternally (cqp/groups.c) and

# SortExternally (cqp/ranges.c)

delete_line_before = list('^\\s*sprintf\\(prefix,\\s"cqpt\\.%d",\\s\\(unsigned\\sint\\)getpid\\(\\)\\);', 1L, 8L),

insert_before = list("^FILE\\s\\*\\s*", "/*", 1),

insert_after = list("^\\}\\s*$", "*/", 3),

replace = list('^(.*?)\\s*/\\*\\sholds\\s"cqpt\\.\\$\\$",\\sso\\s64\\schars\\sis\\splenty\\sof\\sheadroom\\s\\*/', "\\1", 1),

replace = list('^(.*?)(\\s*)/\\*\\s"cqpt\\.\\$\\$"\\s\\*/', "\\1", 1),

replace = list("^(.*?)\\s/\\*\\sstring\\sis\\sallocated\\sby\\stempnam\\(\\),\\sneeds\\sto\\sbe\\sfree'd\\sbelow\\s\\*/", "\\1", 1)

)

),

"src/cwb/cqp/parser.y" = c(

list(),

# Want to see issues with r1690 before adapting / checking these patches

if (revision == 1069) list(

delete_line_before = list("^\\s*if\\s\\(\\$2\\s&&\\sgenerate_code\\)\\s\\{", 3L),

replace = list("^(\\s*)int\\sok;", "\\1int ok __attribute__((unused));", 3),

replace = list("^(\\s*)int\\sok;", "\\1int ok __attribute__((unused));", 2),

replace = list("^(\\s*)ok\\s=\\sSortSubcorpus\\(\\$2,\\s\\$3,\\s\\(\\$4\\s>=\\s1\\)\\s\\?\\s\\$4\\s:\\s1,\\s&\\(\\$5\\)\\);", "\\1SortSubcorpus($2, $3, ($4 >= 1) ? $4 : 1, &($5));", 1)

)

),

"src/cwb/cqp/ranges.c" = c(

list(

# ranges.c: In function 'SortExternally':

# ranges.c:1032:17: warning: 'p1end' may be used uninitialized in this function [-Wmaybe-uninitialized]

# 1032 | else if (p1end >= text_size)

# | ^

# ranges.c:1027:17: warning: 'p1start' may be used uninitialized in this function [-Wmaybe-uninitialized]

# 1027 | else if (p1start >= text_size)

# | ^

insert_after = list("int line, p1start, p1end, plen, step, token, l;", c(" p1start = -1;", " p1end = -1;"), 1L)

),

# This is highly specific - want to see what happens before adapting things to r1690

if (revision == 1069) list(

delete_line_before = list("^\\s*for\\s\\(i\\s=\\s0;\\si\\s<\\sp;\\si\\+\\+\\)", 2L, 5L),

delete_line_before = list("^\\s*for\\s\\(i\\s=\\s0;\\si\\s<\\sp;\\si\\+\\+\\)", 1L, 5L),

delete_line_before = list("^\\s*value\\s=\\scl_string_canonical\\(value,\\ssrt_cl->corpus->charset,\\ssrt_flags,\\sCL_STRING_CANONICAL_STRDUP\\);", 1L, 7L),

insert_before = list("^int", "/*", 9),

insert_after = list("^\\}\\s*$", "*/", 11),

replace = list("^(\\s*)line\\s=\\s-1;(\\s*)\\s/\\*\\swill\\sindicate\\ssort\\sfailure\\sbelow\\sif\\stext_size\\s==\\s0\\s\\*/", "\\1line = -1;\\2", 1),

replace = list("^(\\s*)\\}(\\s)/\\*\\send\\sfor\\seach\\stoken\\s\\*/", "\\1}\\2", 1),

replace = list("^(\\s*)line\\s=\\s-1;(\\s*)/\\*\\swill\\sindicate\\sfailure\\sof\\sexternal\\ssort\\scommand\\s*\\*/", "\\1line = -1;\\2", 1),

replace = list("^(\\s*)break;\\s*/\\*\\sabort\\s\\*/", "\\1break;", 1),

replace = list("^(\\s*)ok\\s=\\sSortExternally\\(\\);", "\\1/* ok = SortExternally(); */", 1),

remove_lines = list("^\\s*/\\*\\suses\\ssettings\\sfrom\\sstatic\\ssrt_\\*\\svariables\\s\\*/", 1),

remove_lines = list("^\\s*/\\*\\sdetermine\\sstart\\sand\\send\\sposition\\sof\\ssort\\sinterval\\sfor\\sthis\\smatch\\s\\*/", 1),

remove_lines = list("/\\*\\sadjust\\ssort\\sboundaries.*?\\*/", 1),

remove_lines = list("/*\\sswap\\sstart\\sand\\send\\sof\\sinterval.*?\\*/", 1),

remove_lines = list("/\\*\\sdetermine\\ssort\\sdirection\\s\\*/", 1),

remove_lines = list("/\\*\\show\\smany\\stokens\\sto\\sprint\\s\\*/", 1),

remove_lines = list("/\\*\\swhen\\susing\\sflags,\\sprint\\snormalised\\stoken\\ssequence.*?\\*/", 1),

remove_lines = list("/\\*\\sprint\\ssequence\\sof\\stokens\\sin\\ssort\\sinterval\\s\\*/", 1),

remove_lines = list("/\\*\\snow,\\sexecute\\sthe\\sexternal\\ssort\\scommand.*?\\*/", 1),

remove_lines = list("/\\*\\srun\\ssort\\scmd\\sand\\sread\\sfrom\\spipe\\s\\*/", 1),

remove_lines = list("/\\*\\snow\\swe\\sshould\\shave\\sread\\sexactly.*?\\*/", 1)

)

),

"src/cwb/cqp/lex.yy.c" = c(

list(),

# See what happens before taking pains to fit this on r1690

if (revision == 1069) list(

# Funktion yyunput auskommentiert, zur Vermeidung von FEhler:

# lex.yy.c:2459:17: warning: unused function 'yyunput' [-Wunused-function]

# auskommentiert: static int input (void)

# um zu vermeiden:

# lex.yy.c:2500:16: warning: function 'input' is not needed and will not be emitted [-Wunneeded-internal-declaration]

delete_line_before = list("/\\*\\*\\sImmediately\\sswitch\\sto\\sa\\sdifferent\\sinput\\sstream\\.", 1L, 111L),

insert_before = list("#ifdef __cplusplus", "/*", 3L),

insert_before = list("/\\*\\*\\sImmediately\\sswitch\\sto\\sa\\sdifferent\\sinput\\sstream\\.", "", 1L),

insert_after = list("#endif", "*/", 27L),