/*  History 
	2026-02-02 Include the names of dummy variable returned in return data
			   set from createDummyVariables.
	2025-12-16 Initial coding
*/

/* Info:
  The main function to create dummy variables for categorical variables is 
    %createDummyVariables(retdata, data, catvars, refcats=, allvars=, maxncats=25, suffix=_CAT, DEBUG=0).
  This function will update the input data set &data by creating dummy variables for the variables
  listed in &catvars and all character variables listed in &allvars. Dummy variables for all 
  non-reference categories will be created. Users can specify the reference categories using the
  &refcats option. If a reference category is not specified for a particular categorical variable,
  then the first category among the sorted list of categories will be designated as the reference.
*/

/* Macro to create dummy variables for categorical variables. 
	Dummy variables will be created if the categorical variable has
	at least two categories and less than or equal to &maxncats categories.
	Any character variable in &allvars is considered to be categorical. 
	If "x" is a categorical variable, then its dummy variables will be called 
	X_CAT0 (reference), X_CAT1, X_CAT2, ...
	INPUT:
	data         (required) The input data set containing all necessary variables
	catvars      (required) Space delimited list of categorical variables. This can be an empty list.
	refcats=     (optional) Space delimited list of the reference categories. The length and order of 
	               this list must match the length and order of catvars.
	allvars=     (optional) Space delimited list of all variables included in the analysis. Dummy variables 
	               will also be created for any character variable in this list which is not in 
				   the catvars list. The reference category will be the first category among
				   the sorted list of all categories.
	maxncats=25  (optional) The maximum number of categories for a categorical variable. This serves as an
	             error check to prevent dummy variables being created for continuous variables
				 or id variables. The default value is 25.
	suffix=_CAT  (optional) The suffix in the dummy variable names that get created. The dummy variables
	             for variable "var" will be called by default VAR_CAT1, VAR_CAT2, ...
	DEBUG=0      (optional) 0 or 1 for debugging purposes. The default is 0.
	
	OUTPUT:
	retdata      (required) A returned data set containing the columns "variable" and "ndummy" which gives
					the name of the categorical variable and number of dummy variables created.
*/
%macro createDummyVariables(retdata, data, catvars, refcats=, allvars=, maxncats=25, suffix=_CAT, DEBUG=0);

	%local ncatvars ncats var cat vars nvars;

    /* Delete returned data if it exists */
	%deleteData(&retdata);

	/* Normalize list of variable names */
	%if &DEBUG %then %put Normalize strings;
	%let catvars  = %normStr(&catvars, upcase=1, removeMultBlanks=1);
	%let allvars  = %normStr(&allvars, upcase=1, removeMultBlanks=1);
	%let allvars  = %getUniqueItemsInList(&allvars);
	
	/* Allvars could contain interactions, so break up those into two variables */
	%let allvars = %sysfunc(tranwrd(&allvars, %str(*), %str( )));
	
	%let refcats  = %normStr(&refcats, upcase=0, removeMultBlanks=1);
	%let ncatvars = %listlen(&catvars);
	%let ncats    = %listlen(&refcats);
	%let nvars    = %listlen(&allvars);
	
    %if %eval((&ncatvars = 0) and (&nvars = 0)) %then %goto exit;

	/* Check that the lengths match */
	%if (&ncatvars) & (&ncats) & (&ncats ~= &ncatvars) %then %do;
		%errorExit(9, Number of categorical variables does not equal the number of reference categories, exit=1);
	%end;
    
	/* Loop over each variable in &catvars and create dummy variables */
	%if %eval(&ncatvars > 0) %then %do;
		%if &DEBUG %then %put Call createDummyVars_loop for catvars;
		%createDummyVars_loop(&retdata, &data, &catvars, refcats=&refcats, maxncats=&maxncats, suffix=&suffix, DEBUG=&DEBUG);
	%end;

	/* Check the variables in allvars. First remove catvars from allvars. */
	%let vars  = %removeItemsFromList(&catvars, &allvars);
	%let nvars = %listlen(&vars);
	%if %eval(&nvars > 0) %then %do;
		/* Determine which of these vars are character */
		%if &DEBUG %then %put Call getCharVarsFromList;
		
		%let vars = %getCharVarsFromList(&data, &vars);
		%if %length(&vars) %then %do;
			%let tmp =;
			%if &DEBUG %then %put Call createDummyVars_loop for other vars;
			%createDummyVars_loop(&retdata, &data, &vars, refcats=&tmp, maxncats=&maxncats, suffix=&suffix, DEBUG=&DEBUG);
		%end;
	%end;
	
	%exit: ;

%mend createDummyVariables;

/* Function to update a list of variables with dummy variables in place of the categorical variables.
   A categorical variable with only one category will be left out.

   INPUT:
   varlist       Space separated list of variables
   catVarsData   The returned data set from %createDummyVariables
   suffix        String to create dummy variable names
   OUTPUT:
                 The updated space delimited list is returned  
 */
%macro updateListWithDummyVars(varlist, catVarsData, suffix=_CAT);

	%local dsid rc nobs ret i j nvars var stop found m vnum1 vnum2;
	
	%let varlist = %normStr(&varlist, upcase=1, removeMultBlanks=1);
	%let nvars   = %listlen(&varlist);
	
	/* See if the data exists */
	%let nobs = %getNobs(&catVarsData, zeroIfMissing=1);
	%if %eval(&nobs < 1) %then %do;
		/* No categorical variables, so return varlist */
		%let ret = &varlist;
	%end;
	%else %if %eval(&nvars < 1) %then %do;
		/* No variables in list, so return varlist */
		%let ret = &varlist;
	%end;
	%else %do;
		/* Initialize return list */
		%let ret=;
	
		/* Store the categorical variables and number of dummy variables for each categorical
		variable in macro variables */
		%let dsid  = %sysfunc(open(&catVarsData));
		%let vnum1 = %sysfunc(varnum(&dsid, variable));
		%let vnum2 = %sysfunc(varnum(&dsid, ndummy));
		%if %eval(&vnum1 <= 0) or %eval(&vnum2 <= 0) %then %do;
			%let rc = %sysfunc(close(&dsid));
			%errorExit(9, INTERNAL CODING ERROR with &catVarsData, exit=1);
		%end;		
		%do i = 1 %to &nobs;
			%local v&i n&i;
			%let rc  = %sysfunc(fetchobs(&dsid, &i));
			%let v&i = %normStr(%sysfunc(getvarc(&dsid, &vnum1)), upcase=1, removeMultBlanks=0);
			%let n&i = %normStr(%sysfunc(getvarn(&dsid, &vnum2)), upcase=0, removeMultBlanks=0);
		%end;
		%let rc = %sysfunc(close(&dsid));
		
		/* Loop of each variable in varlist and replace a categorical variable with dummy variables,
			preserving the order of varlist */
		%do i = 1 %to &nvars;
			%let var   = %scan(&varlist, &i, ' ');
			%let j     = 0;
			%let stop  = 0;
			%let found = 0;
			%do %until (&stop);
				%let j = %eval(&j + 1);
				%if ("&var" = "&&v&j") %then %do;
					/* Variable found in list of categorical variables */
					%let found = &j;
					%let stop  = 1;
				%end;
				%if %eval(&j >= &nobs) %then %let stop = 1;
			%end;
			/* If found, then check the number of dummy vars. If ndummy = 0, then this
				variable will not be included in return list */
			%if &found %then %do;
				%let m = &&n&found;
				%if %eval(&m > 0) %then %do j = 1 %to &m;
					%let ret = &ret &var.&suffix.&j;
				%end;
			%end;
			%else %do;
				/* Not categorical, so keep original variable in return list */
				%let ret = &ret &var;
			%end;
		%end;
		
	%end;
	%if %length(&ret) %then %let ret = %normStr(&ret, upcase=1, removeMultBlanks=1);

	&ret

%mend updateListWithDummyVars;

/* Macro to update a SAS data set of covariate lists with dummy variables. 
   This macro updates variable &var of the input data set &data. Also, the
   length of variable &var will be increased.
   INPUT:
   data          The data set to be updated containing variable var
   catVarsData   The returned data set from %createDummyVariables
   var           The variable to be updated. The default is covariates
   suffix        String to create dummy variable names
   OUTPUT:
                 The data set &data is updated 
*/
%macro updateDataWithDummyVars(data, catVarsData, var="covariates", suffix=_CAT);

	%local nobs len newlen i covlist newlist;

	%let nobs = %getNobs(&data, zeroIfMissing=1);
	%if &nobs %then %do;
		** We probably need to increase the LENGTH of the SAS variable;
		%let len    = %getVarLen(&data, &var);
		%let newlen = %sysfunc(min(&len + 1000, 32767));
		
		** Loop over each row of data;
		%do i = 1 %to &nobs;
			%let covlist =;
			** Get the ith covariate list;
			data _NULL_; set &data ;
				if (_N_ = &i) then call symputx("covlist", &var);
			run;
			%let newlist = %updateListWithDummyVars(&covlist, &catVarsData, suffix=&suffix);
			%if %length(&newlist) %then %do;
				data &data; 
					length &var $ &newlen;
					set &data;
					if (_N_ = &i) then &var = "&newlist"; 
				run;
			%end;
		%end;
	%end;

%mend updateDataWithDummyVars;

%macro createDummyVars_loop(retdata, data, catvars, refcats, maxncats=25, suffix=_CAT, DEBUG=0);

	%local ncatvars ncats i var cat tmp tmpdata refFlag j dvars;
	
	/* Temporary data set */
	%let tmpdata  = _crDumVarlp_tmp;
	%let ncatvars = %listlen(&catvars);
	%let refFlag  = %length(%normStr(&refcats, upcase=0, removeMultBlanks=0));

	%do i = 1 %to &ncatvars;
	
		%let var = %left(%trim(%scan(&catvars, &i, ' ')));
		%if &DEBUG %then %put variable = &var;
		%let dvars = ;
		
		** Check that the variable is in the data;
		%let tmp = %getVarNum(&data, &var);
		%if %eval(&tmp = 0) %then %do;
			%errorExit(9, Variable &var no found in data, exit=1);
		%end;	
		
		** Check to see if dummy vars have not already been created;
		%let tmp = 0; /* Assume dummy vars have not been created */
		%if %sysfunc(exist(&retdata, DATA)) %then %do; 
			data _NULL_; set &retdata;
				if variable = "&var" then do;
					call symputx("tmp", 1);
					stop;
				end;
			run;	
		%end;
		%if %eval(&tmp = 0) %then %do;
			** Get the distinct categories;
			%deleteData(&tmpdata);
			%getUnqVals(&data, &var, &tmpdata);
			%let ncats = %getNobs(&tmpdata);
						
			** Check the number of categories;
			%if %eval((&ncats > 1) and (&ncats <= &maxncats)) %then %do;
				** Get the reference category and remove it from tmpdata;
				%if &refFlag %then %let cat = %left(%trim(%scan(&refcats, &i, ' ')));
				%else              %let cat =;
				%if &DEBUG %then %put reference = &cat;			
				%removeCatFromData(&tmpdata, &var, &cat);	/* error is here */			
				** Create the dummy variables;
				%createDummy(&data, &var, &tmpdata, suffix=&suffix);
			%end;
			%else %if %eval(&ncats <= 1) %then %do;
				%put WARNING: Variable &var has only one category and will be removed from the analysis;
			%end;
			%else %if %eval(&ncats > &maxncats) %then %do;
				%errorExit(9, Variable &var has too many categories. Change maxncats or review the variable., exit=1);
			%end;
			
			** Save information;
			%let ncats = %eval(&ncats - 1);
			%if %eval(&ncats > 0) %then %do;
				%do j = 1 %to &ncats;
					%let dvars = &dvars	&var.&suffix.&j;
				%end;
			%end;
			data &tmpdata;
				length variable $ 100;
				length dummy_vars $ 1000;
				variable   = "&var";
				ndummy     = &ncats;
				dummy_vars = left(trim("&dvars"));
			run;
			%append(&retdata, &tmpdata);
			
		%end;
	
	%end;

%mend createDummyVars_loop;

/* Macro to remove a category from a data set of categories. If the category is not found,
	then the first category will be dropped and a warning produced.
	The input data set is updated. */
%macro removeCatFromData(data, var, cat);

	%local nobs0 nobs1 ischar cat1 catFlag;
	
	%let catFlag = %length(&cat);
	
	/* Remove the category &cat, if cat was specified */
	%if &catFlag %then %do;
		%let nobs0   = %getNobs(&data);
		%let ischar  = %isCharVar(&data, &var); 

		%if &ischar %then %do;
			data &data; set &data;
				if (&var = "&cat") then delete;
				else output;
			run;
		%end;
		%else %do;
			data &data; set &data;
				if (&var = &cat) then delete;
				else output;
			run;
		%end;
	
		%let nobs1 = %getNobs(&data);
	%end;
	%else %do;
		%let nobs0 = 0;
		%let nobs1 = 0;
	%end;
	
	/* Determine if a category was actually removed */
	%if %eval(&nobs0 - &nobs1 = 0) %then %do;
		** Category was not found, so remove the first category;
		data &data; set &data;
			if (_N_ = 1) then do;
				CALL symputx("cat1", &var);
				delete;
			end;
			else output;
		run;
		%if &refFlag %then %put WARNING: Reference category &cat not found for variable &var., &cat1 will be the reference;
	%end;

%mend removeCatFromData;

/* Create dummy vars on input data. The dummy variables will be &var_cat1, &var_cat2, ... 	
	The input data set will be updated.
	data        The input and output data set
	var         Name of the categorical variable
	datacats    Data set with the variable &var containing the categories to create
	              dummy variables. The out data set from %getUnqVals(data, var, out)
	suffix
*/
%macro createDummy(data, var, datacats, suffix=_cat);

	%local i ncats ischar;

	%let ncats  = %getNobs(&datacats);
	%let ischar = %isCharVar(&data, &var);

	/* Store categories in macro variables */
	data _NULL_; set &datacats;
		call symputx(cats("cat", _N_), &var);
	run;

	/* Add dummy vars to data */
	data &data; set &data;
		%do i = 1 %to &ncats;
			&var.&suffix.&i = 0;
		%end;
		%if &ischar %then %do;
			%do i = 1 %to &ncats;
				if (&var = "&&cat&i") then &var.&suffix.&i = 1;
			%end;
		%end;
		%else %do;
			%do i = 1 %to &ncats;
				if (&var = &&cat&i) then &var.&suffix.&i = 1;
			%end;
		%end;
	run;

%mend createDummy;

/* Function to remove items from a list and return the updated list */
%macro removeItemsFromList(items, list);

  %local Nlist Nitems i j ret itemi listj flag;

  %let Nlist  = %listlen(&list);
  %let Nitems = %listlen(&items);
  %let ret    = ; /* Assume list is empty, then return nothing */

  %if %eval(&Nlist > 0) and %eval(&Nitems > 0) %then %do;    
    /* Both are non-empty */
    %do j = 1 %to &Nlist;
      %let listj = %normStr(%scan(&list, &j, ' '));
      %let flag  = 0;
      %do i = 1 %to &Nitems;
        %let itemi = %normStr(%scan(&items, &i, ' '));
        %if ("&itemi" = "&listj") %then %let flag = 1;
      %end;
      %if %eval(&flag = 0) %then %let ret = &ret &listj;
    %end;
  %end;
  %else %if %eval(&Nlist > 0) %then %let ret = &list; /* items is empty */

  &ret

%mend removeItemsFromList;

%macro itemInList(list, item, N=-1);

  %local i stop ret tmp N;

  %let ret  = 0; /* Assume item is not in the list */
  %if (%eval(&N < 0)) %then %let N = %listlen(&list);
  %if (&N > 0) %then %do;
    %let stop = 0;
    %let i    = 1;
    %do %until (&stop = 1);
      %let tmp = %left(%trim(%scan(&list, &i, ' ')));
      %if ("&tmp" = "&item") %then %do;
        %let ret = 1; /* item is in list */
        %let stop = 1;
      %end;
      %let i = %eval(&i + 1);
      %if (&i > &N) %then %let stop = 1;
    %end; 
  %end;

  &ret

%mend itemInList;

%macro getUniqueItemsInList(list);

	%local i j ret len itemi retlen notfound;
	
	%let list = %normStr(&list, upcase=0, removeMultBlanks=1);
	%let len  = %listlen(&list);
	%if %eval(&len = 0) %then %do;
		%let ret = &list;
	%end;
	%else %do i = 1 %to &len;
		%let itemi = %scan(&list, &i, ' ');
		%if %eval(&i = 1) %then %do;
		    %local used1;
			%let ret    = &itemi;
			%let retlen = 1;
			%let used1  = &itemi;
		%end;
		%else %do;
			%let notfound = 1;
			%do j = 1 %to &retlen;
				%if ("&itemi" = "&&used&j") %then %let notfound = 0;
			%end;
			%if &notfound %then %do;
				%let retlen = %eval(&retlen + 1);
				%local used&retlen;
				%let ret         = &ret &itemi;
				%let used&retlen = &itemi;
			%end;
		%end;
	%end;
	
	&ret

%mend getUniqueItemsInList;

/* Macro to normalize character strings */
%macro normStr(inStr, upcase=1, removeMultBlanks=1);
  /* Macro to normalize a character string 

  */
  %local __ret;
  %let __ret = %trim(%left(&inStr));
  %if %length(&__ret) %then %do;
    %if (&removeMultBlanks) %then %let __ret = %sysfunc(compbl(&__ret));
    %if (&upcase) %then %let __ret = %upcase(&__ret);
  %end;
  &__ret

%mend normStr;

/* Macro to get the number of items in a space delimited list */
%macro listLen(list);
  
  %local len;

  %if (%length(&list) = 0) %then %let len = 0;
  %else %do;
    %let list = %normStr(&list, upcase=0, removeMultBlanks=1);
    %let len  = %eval(%sysfunc(count(&list, %str( ))) + 1);
  %end;
  
  &len

%mend listLen;

%macro deleteData(data, type=DATA);
  /* Macro to delete a data set.
     data   Sas data set to be deleted. No default.
     type   DATA or VIEW
  */
  %if %sysfunc(exist(&data, &type)) %then %do;
    %if (%upcase(&type) ^= VIEW) %then %let type = TABLE;
    proc sql; drop &type &data; quit;
  %end;
%mend deleteData;

%macro getNobs(data, zeroIfMissing=1);

  %local dsid nobs rc;

  %let data = %normStr(&data, upcase=0, removeMultBlanks=0);
  %let nobs = -9999;  /* Missing */
  %if %length(&data) %then %let rc = %sysfunc(exist(&data, DATA));
  %else                    %let rc = 0;
  %if (&rc) %then %do;
    %let dsid = %sysfunc(open(&data));
    %let nobs = %sysfunc(attrn(&dsid, NOBS));
    %let rc   = %sysfunc(close(&dsid)); 
  %end;
  %else %do;
    %if (&zeroIfMissing) %then %let nobs = 0;
  %end;

  &nobs

%mend getNobs;

%macro getVarNum(data, var);

  %local dsid rc ret;
  
  %let ret  = 0;
  %let dsid = %sysfunc(open(&data));
  %let ret  = %sysfunc(varnum(&dsid, &var));
  %let rc   = %sysfunc(close(&dsid));
  &ret

%mend getVarNum;

%macro getVarLen(data, var);

  %local dsid rc ret vnum;
  
  %let ret  = 0;
  %let dsid = %sysfunc(open(&data));
  %let vnum = %sysfunc(varnum(&dsid, &var));
  %let ret  = %sysfunc(varlen(&dsid, &vnum));
  %let rc   = %sysfunc(close(&dsid));
  &ret

%mend getVarLen;

/* Type of variable (N, C, or E for error) */
%macro getVarType(data, vars);

  %local dsid rc vnum type nvars i var ret;

  %let nvars = %listlen(&vars);
  %let ret   = E;
  %if %eval(&nvars > 0) %then %do;
	%let dsid = %sysfunc(open(&data));
	%do i = 1 %to &nvars;
	  %let var  = %scan(&vars, &i, ' ');
      %let vnum = %sysfunc(varnum(&dsid, &var));
	  %if &vnum %then %let type = %sysfunc(vartype(&dsid, &vnum));
	  %else           %let type = E; /* Variable not on data */
      %if %eval(&i = 1) %then %let ret = &type;		
	  %else                   %let ret = &ret &type;
	%end;
    %let rc = %sysfunc(close(&dsid));
  %end;
  
  &ret

%mend getVarType;

/* Function to determine the character variables from a list of variables */
%macro isCharVar(data, vars);
	
	%local i types type ret nvars tmp;
	
	%let types = %getVarType(&data, &vars);
	%let nvars = %listlen(&vars);

	%if %eval(&nvars > 0) %then %do;
		%do i = 1 %to &nvars;
			%let tmp  = 0;
			%let type = %normStr(%scan(&types, &i, ' '));
			%if %str(&type) = %str(C) %then %let tmp = 1;
			%if %eval(&i = 1) %then %let ret = &tmp;
			%else                   %let ret = &ret &tmp;
		%end;
	%end;	

	&ret

%mend isCharVar;

/* Function to get the character vars from a list of vars */
%macro getCharVarsFromList(data, vars);

	%local ret i types type nvars var;

	%let ret   =;
	%let nvars = %listlen(&vars);
	%if %eval(&nvars > 0) %then %do;
		%let types = %getVarType(&data, &vars);
		%do i = 1 %to &nvars;
			%let type = %normStr(%scan(&types, &i, ' '));
			%if %str(&type) =%str(C) %then %do;
				%let var = %scan(&vars, &i, ' ');
				%if %length(&ret) %then %let ret = &ret &var;
				%else %let ret = &var;
			%end;
		%end;
	%end;
	
	&ret

%mend getCharVarsFromList;

/* Function to get unique values of a variable */
%macro getUnqVals(data, var, out);

  %local tmpdata flag;

  %let data = %normStr(&data);
  %let out  = %normStr(&out);

  /* To remove warning message from proc sql when data=out */
  %if ("&data" = "&out") %then %do;
    %let flag    = 1;
    %let tmpdata = &out._sqlunq;
  %end;
  %else %do;
    %let flag = 0;
    %let tmpdata = &out;
  %end;

  proc sql noprint; 
    create table &tmpdata as select distinct * 
    from &data(keep=&var) order by &var;
  quit;
  %if &flag %then %do;
    data &out; set &tmpdata; run;
    %deleteData(&tmpdata);
  %end;


%mend getUnqVals;

%macro append(baseData, newData, baseClause=, newClause=, rowNum=);
   /* Appends a sas data set to another. The data sets baseData and newData
      must have the same variables in common.
      baseData     Base sas data set. No default.
      newData      Sas data set to append. No default.
      baseClause   Data set clause for baseData.
      newClause    Data set clause for newData.
      rowNum       Variable to add row numbers
   */

   %local nobs rowFlag tmpdata;
   %let rowNum  = %left(%trim(&rowNum));
   %let rowFlag = %length(&rowNum);
   %let tmpdata = _append_tmp_;

   %if %sysfunc(exist(&newData)) %then %do;
     %if %sysfunc(exist(&baseData)) %then %do;
	    %if &rowFlag %then %do;
          %let nobs = %getNobs(&baseData);
		  data &tmpdata; set &newData&newClause;
            &rowNum = &nobs + _N_;
		  run;
		%end;
		%else %let tmpdata = &newData&newClause;
        proc append base=&baseData&baseClause data=&tmpdata force; run;
     %end;
     %else %do;
       data &baseData&baseClause; set &newData&newClause; 
       %if &rowFlag %then %do;
         &rowNum = _N_;
	   %end;
       run;
     %end;
   %end;

   %if &rowFlag %then %deleteData(&tmpdata);

%mend append;

