root/tools/crm_resource_runtime.c

/* */

DEFINITIONS

1. build_node_info_list
2. cli_resource_search
3. find_resource_attr
4. find_matching_attr_resources_recursive
5. find_matching_attr_resources
6. cli_resource_update_attribute
7. cli_resource_delete_attribute
8. send_lrm_rsc_op
9. rsc_fail_name
10. clear_rsc_history
11. clear_rsc_failures
12. clear_rsc_fail_attrs
13. cli_resource_delete
14. cli_cleanup_all
15. check_role
16. check_managed
17. check_locked
18. node_is_unhealthy
19. check_node_health
20. cli_resource_check
21. cli_resource_fail
22. generate_resource_params
23. resource_is_running_on
24. get_active_resources
25. dump_list
26. display_list
27. update_working_set_xml
28. update_working_set_from_cib
29. update_dataset
30. max_delay_for_resource
31. max_delay_in
32. cli_resource_restart
33. action_is_pending
34. actions_are_pending
35. print_pending_actions
36. wait_till_stable
37. get_action
38. set_agent_environment
39. apply_overrides
40. cli_resource_execute_from_params
41. cli_resource_execute
42. cli_resource_move

   1 /*
   2  * Copyright 2004-2023 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <crm_resource.h>
  13 #include <crm/common/ipc_attrd_internal.h>
  14 #include <crm/common/ipc_controld.h>
  15 #include <crm/common/lists_internal.h>
  16 #include <crm/services_internal.h>
  17 
  18 static GList *
  19 build_node_info_list(const pe_resource_t *rsc)
     /*  */
  20 {
  21     GList *retval = NULL;
  22 
  23     for (const GList *iter = rsc->children; iter != NULL; iter = iter->next) {
  24         const pe_resource_t *child = (const pe_resource_t *) iter->data;
  25 
  26         for (const GList *iter2 = child->running_on;
  27              iter2 != NULL; iter2 = iter2->next) {
  28 
  29             const pe_node_t *node = (const pe_node_t *) iter2->data;
  30             node_info_t *ni = calloc(1, sizeof(node_info_t));
  31 
  32             ni->node_name = node->details->uname;
  33             ni->promoted = pcmk_is_set(rsc->flags, pe_rsc_promotable) &&
  34                            child->fns->state(child, TRUE) == RSC_ROLE_PROMOTED;
  35 
  36             retval = g_list_prepend(retval, ni);
  37         }
  38     }
  39 
  40     return retval;
  41 }
  42 
  43 GList *
  44 cli_resource_search(pe_resource_t *rsc, const char *requested_name,
     /*  */
  45                     pe_working_set_t *data_set)
  46 {
  47     GList *retval = NULL;
  48     const pe_resource_t *parent = pe__const_top_resource(rsc, false);
  49 
  50     if (pe_rsc_is_clone(rsc)) {
  51         retval = build_node_info_list(rsc);
  52 
  53     /* The anonymous clone children's common ID is supplied */
  54     } else if (pe_rsc_is_clone(parent)
  55                && !pcmk_is_set(rsc->flags, pe_rsc_unique)
  56                && rsc->clone_name
  57                && pcmk__str_eq(requested_name, rsc->clone_name, pcmk__str_casei)
  58                && !pcmk__str_eq(requested_name, rsc->id, pcmk__str_casei)) {
  59 
  60         retval = build_node_info_list(parent);
  61 
  62     } else if (rsc->running_on != NULL) {
  63         for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
  64             pe_node_t *node = (pe_node_t *) iter->data;
  65             node_info_t *ni = calloc(1, sizeof(node_info_t));
  66             ni->node_name = node->details->uname;
  67             ni->promoted = (rsc->fns->state(rsc, TRUE) == RSC_ROLE_PROMOTED);
  68 
  69             retval = g_list_prepend(retval, ni);
  70         }
  71     }
  72 
  73     return retval;
  74 }
  75 
  76 // \return Standard Pacemaker return code
  77 static int
  78 find_resource_attr(pcmk__output_t *out, cib_t * the_cib, const char *attr,
     /*  */
  79                    const char *rsc, const char *attr_set_type, const char *set_name,
  80                    const char *attr_id, const char *attr_name, char **value)
  81 {
  82     int rc = pcmk_rc_ok;
  83     xmlNode *xml_search = NULL;
  84     GString *xpath = NULL;
  85     const char *xpath_base = NULL;
  86 
  87     if(value) {
  88         *value = NULL;
  89     }
  90 
  91     if(the_cib == NULL) {
  92         return ENOTCONN;
  93     }
  94 
  95     xpath_base = pcmk_cib_xpath_for(XML_CIB_TAG_RESOURCES);
  96     if (xpath_base == NULL) {
  97         crm_err(XML_CIB_TAG_RESOURCES " CIB element not known (bug?)");
  98         return ENOMSG;
  99     }
 100 
 101     xpath = g_string_sized_new(1024);
 102     pcmk__g_strcat(xpath,
 103                    xpath_base, "//*[@" XML_ATTR_ID "=\"", rsc, "\"]", NULL);
 104 
 105     if (attr_set_type != NULL) {
 106         pcmk__g_strcat(xpath, "/", attr_set_type, NULL);
 107         if (set_name != NULL) {
 108             pcmk__g_strcat(xpath, "[@" XML_ATTR_ID "=\"", set_name, "\"]",
 109                            NULL);
 110         }
 111     }
 112 
 113     g_string_append(xpath, "//" XML_CIB_TAG_NVPAIR "[");
 114     if (attr_id != NULL) {
 115         pcmk__g_strcat(xpath, "@" XML_ATTR_ID "=\"", attr_id, "\"", NULL);
 116     }
 117 
 118     if (attr_name != NULL) {
 119         if (attr_id != NULL) {
 120             g_string_append(xpath, " and ");
 121         }
 122         pcmk__g_strcat(xpath, "@" XML_NVPAIR_ATTR_NAME "=\"", attr_name, "\"",
 123                        NULL);
 124     }
 125     g_string_append_c(xpath, ']');
 126 
 127     rc = the_cib->cmds->query(the_cib, (const char *) xpath->str, &xml_search,
 128                               cib_sync_call | cib_scope_local | cib_xpath);
 129     rc = pcmk_legacy2rc(rc);
 130 
 131     if (rc != pcmk_rc_ok) {
 132         goto done;
 133     }
 134 
 135     crm_log_xml_debug(xml_search, "Match");
 136     if (xml_has_children(xml_search)) {
 137         xmlNode *child = NULL;
 138 
 139         rc = ENOTUNIQ;
 140         out->info(out, "Multiple attributes match name=%s", attr_name);
 141 
 142         for (child = pcmk__xml_first_child(xml_search); child != NULL;
 143              child = pcmk__xml_next(child)) {
 144             out->info(out, "  Value: %s \t(id=%s)",
 145                       crm_element_value(child, XML_NVPAIR_ATTR_VALUE), ID(child));
 146         }
 147 
 148         out->spacer(out);
 149 
 150     } else if(value) {
 151         pcmk__str_update(value, crm_element_value(xml_search, attr));
 152     }
 153 
 154   done:
 155     g_string_free(xpath, TRUE);
 156     free_xml(xml_search);
 157     return rc;
 158 }
 159 
 160 /* PRIVATE. Use the find_matching_attr_resources instead. */
 161 static void
 162 find_matching_attr_resources_recursive(pcmk__output_t *out, GList/* <pe_resource_t*> */ ** result,
     /*  */
 163                                        pe_resource_t * rsc, const char * rsc_id,
 164                                        const char * attr_set, const char * attr_set_type,
 165                                        const char * attr_id, const char * attr_name,
 166                                        cib_t * cib, const char * cmd, int depth)
 167 {
 168     int rc = pcmk_rc_ok;
 169     char *lookup_id = clone_strip(rsc->id);
 170     char *local_attr_id = NULL;
 171 
 172     /* visit the children */
 173     for(GList *gIter = rsc->children; gIter; gIter = gIter->next) {
 174         find_matching_attr_resources_recursive(out, result, (pe_resource_t*)gIter->data,
 175                                                rsc_id, attr_set, attr_set_type,
 176                                                attr_id, attr_name, cib, cmd, depth+1);
 177         /* do it only once for clones */
 178         if(pe_clone == rsc->variant) {
 179             break;
 180         }
 181     }
 182 
 183     rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
 184                             attr_set, attr_id, attr_name, &local_attr_id);
 185     /* Post-order traversal. 
 186      * The root is always on the list and it is the last item. */
 187     if((0 == depth) || (pcmk_rc_ok == rc)) {
 188         /* push the head */
 189         *result = g_list_append(*result, rsc);
 190     }
 191 
 192     free(local_attr_id);
 193     free(lookup_id);
 194 }
 195 
 196 
 197 /* The result is a linearized pre-ordered tree of resources. */
 198 static GList/*<pe_resource_t*>*/ *
 199 find_matching_attr_resources(pcmk__output_t *out, pe_resource_t * rsc,
     /*  */
 200                              const char * rsc_id, const char * attr_set,
 201                              const char * attr_set_type, const char * attr_id,
 202                              const char * attr_name, cib_t * cib, const char * cmd,
 203                              gboolean force)
 204 {
 205     int rc = pcmk_rc_ok;
 206     char *lookup_id = NULL;
 207     char *local_attr_id = NULL;
 208     GList * result = NULL;
 209     /* If --force is used, update only the requested resource (clone or primitive).
 210      * Otherwise, if the primitive has the attribute, use that.
 211      * Otherwise use the clone. */
 212     if(force == TRUE) {
 213         return g_list_append(result, rsc);
 214     }
 215     if(rsc->parent && pe_clone == rsc->parent->variant) {
 216         int rc = pcmk_rc_ok;
 217         char *local_attr_id = NULL;
 218         rc = find_resource_attr(out, cib, XML_ATTR_ID, rsc_id, attr_set_type,
 219                                 attr_set, attr_id, attr_name, &local_attr_id);
 220         free(local_attr_id);
 221 
 222         if(rc != pcmk_rc_ok) {
 223             rsc = rsc->parent;
 224             out->info(out, "Performing %s of '%s' on '%s', the parent of '%s'",
 225                       cmd, attr_name, rsc->id, rsc_id);
 226         }
 227         return g_list_append(result, rsc);
 228     } else if(rsc->parent == NULL && rsc->children && pe_clone == rsc->variant) {
 229         pe_resource_t *child = rsc->children->data;
 230 
 231         if(child->variant == pe_native) {
 232             lookup_id = clone_strip(child->id); /* Could be a cloned group! */
 233             rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
 234                                     attr_set, attr_id, attr_name, &local_attr_id);
 235 
 236             if(rc == pcmk_rc_ok) {
 237                 rsc = child;
 238                 out->info(out, "A value for '%s' already exists in child '%s', performing %s on that instead of '%s'",
 239                           attr_name, lookup_id, cmd, rsc_id);
 240             }
 241 
 242             free(local_attr_id);
 243             free(lookup_id);
 244         }
 245         return g_list_append(result, rsc);
 246     }
 247     /* If the resource is a group ==> children inherit the attribute if defined. */
 248     find_matching_attr_resources_recursive(out, &result, rsc, rsc_id, attr_set,
 249                                            attr_set_type, attr_id, attr_name,
 250                                            cib, cmd, 0);
 251     return result;
 252 }
 253 
 254 // \return Standard Pacemaker return code
 255 int
 256 cli_resource_update_attribute(pe_resource_t *rsc, const char *requested_name,
     /*  */
 257                               const char *attr_set, const char *attr_set_type,
 258                               const char *attr_id, const char *attr_name,
 259                               const char *attr_value, gboolean recursive,
 260                               cib_t *cib, int cib_options, gboolean force)
 261 {
 262     pcmk__output_t *out = rsc->cluster->priv;
 263     int rc = pcmk_rc_ok;
 264 
 265     char *found_attr_id = NULL;
 266 
 267     GList/*<pe_resource_t*>*/ *resources = NULL;
 268     const char *top_id = pe__const_top_resource(rsc, false)->id;
 269 
 270     if ((attr_id == NULL) && !force) {
 271         find_resource_attr(out, cib, XML_ATTR_ID, top_id, NULL, NULL, NULL,
 272                            attr_name, NULL);
 273     }
 274 
 275     if (pcmk__str_eq(attr_set_type, XML_TAG_ATTR_SETS, pcmk__str_casei)) {
 276         if (!force) {
 277             rc = find_resource_attr(out, cib, XML_ATTR_ID, top_id,
 278                                     XML_TAG_META_SETS, attr_set, attr_id,
 279                                     attr_name, &found_attr_id);
 280             if ((rc == pcmk_rc_ok) && !out->is_quiet(out)) {
 281                 out->err(out,
 282                          "WARNING: There is already a meta attribute "
 283                          "for '%s' called '%s' (id=%s)",
 284                          top_id, attr_name, found_attr_id);
 285                 out->err(out,
 286                          "         Delete '%s' first or use the force option "
 287                          "to override", found_attr_id);
 288             }
 289             free(found_attr_id);
 290             if (rc == pcmk_rc_ok) {
 291                 return ENOTUNIQ;
 292             }
 293         }
 294         resources = g_list_append(resources, rsc);
 295 
 296     } else if (pcmk__str_eq(attr_set_type, ATTR_SET_ELEMENT, pcmk__str_none)) {
 297         crm_xml_add(rsc->xml, attr_name, attr_value);
 298         CRM_ASSERT(cib != NULL);
 299         rc = cib->cmds->replace(cib, XML_CIB_TAG_RESOURCES, rsc->xml,
 300                                 cib_options);
 301         rc = pcmk_legacy2rc(rc);
 302         if (rc == pcmk_rc_ok) {
 303             out->info(out, "Set attribute: name=%s value=%s",
 304                       attr_name, attr_value);
 305         }
 306         return rc;
 307 
 308     } else {
 309         resources = find_matching_attr_resources(out, rsc, requested_name,
 310                                                  attr_set, attr_set_type,
 311                                                  attr_id, attr_name, cib,
 312                                                  "update", force);
 313     }
 314 
 315     /* If the user specified attr_set or attr_id, the intent is to modify a
 316      * single resource, which will be the last item in the list.
 317      */
 318     if ((attr_set != NULL) || (attr_id != NULL)) {
 319         GList *last = g_list_last(resources);
 320 
 321         resources = g_list_remove_link(resources, last);
 322         g_list_free(resources);
 323         resources = last;
 324     }
 325 
 326     for (GList *iter = resources; iter != NULL; iter = iter->next) {
 327         char *lookup_id = NULL;
 328         char *local_attr_set = NULL;
 329         const char *rsc_attr_id = attr_id;
 330         const char *rsc_attr_set = attr_set;
 331 
 332         xmlNode *xml_top = NULL;
 333         xmlNode *xml_obj = NULL;
 334         found_attr_id = NULL;
 335 
 336         rsc = (pe_resource_t *) iter->data;
 337 
 338         lookup_id = clone_strip(rsc->id); /* Could be a cloned group! */
 339         rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
 340                                 attr_set, attr_id, attr_name, &found_attr_id);
 341 
 342         switch (rc) {
 343             case pcmk_rc_ok:
 344                 crm_debug("Found a match for name=%s: id=%s",
 345                           attr_name, found_attr_id);
 346                 rsc_attr_id = found_attr_id;
 347                 break;
 348 
 349             case ENXIO:
 350                 if (rsc_attr_set == NULL) {
 351                     local_attr_set = crm_strdup_printf("%s-%s", lookup_id,
 352                                                        attr_set_type);
 353                     rsc_attr_set = local_attr_set;
 354                 }
 355                 if (rsc_attr_id == NULL) {
 356                     found_attr_id = crm_strdup_printf("%s-%s",
 357                                                       rsc_attr_set, attr_name);
 358                     rsc_attr_id = found_attr_id;
 359                 }
 360 
 361                 xml_top = create_xml_node(NULL, crm_element_name(rsc->xml));
 362                 crm_xml_add(xml_top, XML_ATTR_ID, lookup_id);
 363 
 364                 xml_obj = create_xml_node(xml_top, attr_set_type);
 365                 crm_xml_add(xml_obj, XML_ATTR_ID, rsc_attr_set);
 366                 break;
 367 
 368             default:
 369                 free(lookup_id);
 370                 free(found_attr_id);
 371                 g_list_free(resources);
 372                 return rc;
 373         }
 374 
 375         xml_obj = crm_create_nvpair_xml(xml_obj, rsc_attr_id, attr_name,
 376                                         attr_value);
 377         if (xml_top == NULL) {
 378             xml_top = xml_obj;
 379         }
 380 
 381         crm_log_xml_debug(xml_top, "Update");
 382 
 383         rc = cib->cmds->modify(cib, XML_CIB_TAG_RESOURCES, xml_top,
 384                                cib_options);
 385         rc = pcmk_legacy2rc(rc);
 386         if (rc == pcmk_rc_ok) {
 387             out->info(out, "Set '%s' option: id=%s%s%s%s%s value=%s",
 388                       lookup_id, found_attr_id,
 389                       ((rsc_attr_set == NULL)? "" : " set="),
 390                       pcmk__s(rsc_attr_set, ""),
 391                       ((attr_name == NULL)? "" : " name="),
 392                       pcmk__s(attr_name, ""), attr_value);
 393         }
 394 
 395         free_xml(xml_top);
 396 
 397         free(lookup_id);
 398         free(found_attr_id);
 399         free(local_attr_set);
 400 
 401         if (recursive
 402             && pcmk__str_eq(attr_set_type, XML_TAG_META_SETS,
 403                             pcmk__str_casei)) {
 404             GList *lpc = NULL;
 405             static bool need_init = true;
 406 
 407             if (need_init) {
 408                 need_init = false;
 409                 pcmk__unpack_constraints(rsc->cluster);
 410                 pe__clear_resource_flags_on_all(rsc->cluster,
 411                                                 pe_rsc_detect_loop);
 412             }
 413 
 414             /* We want to set the attribute only on resources explicitly
 415              * colocated with this one, so we use rsc->rsc_cons_lhs directly
 416              * rather than the with_this_colocations() method.
 417              */
 418             pe__set_resource_flags(rsc, pe_rsc_detect_loop);
 419             for (lpc = rsc->rsc_cons_lhs; lpc != NULL; lpc = lpc->next) {
 420                 pcmk__colocation_t *cons = (pcmk__colocation_t *) lpc->data;
 421 
 422                 crm_debug("Checking %s %d", cons->id, cons->score);
 423                 if (!pcmk_is_set(cons->dependent->flags, pe_rsc_detect_loop)
 424                     && (cons->score > 0)) {
 425                     crm_debug("Setting %s=%s for dependent resource %s",
 426                               attr_name, attr_value, cons->dependent->id);
 427                     cli_resource_update_attribute(cons->dependent,
 428                                                   cons->dependent->id, NULL,
 429                                                   attr_set_type, NULL,
 430                                                   attr_name, attr_value,
 431                                                   recursive, cib, cib_options,
 432                                                   force);
 433                 }
 434             }
 435         }
 436     }
 437     g_list_free(resources);
 438     return rc;
 439 }
 440 
 441 // \return Standard Pacemaker return code
 442 int
 443 cli_resource_delete_attribute(pe_resource_t *rsc, const char *requested_name,
     /*  */
 444                               const char *attr_set, const char *attr_set_type,
 445                               const char *attr_id, const char *attr_name,
 446                               cib_t *cib, int cib_options, gboolean force)
 447 {
 448     pcmk__output_t *out = rsc->cluster->priv;
 449     int rc = pcmk_rc_ok;
 450     GList/*<pe_resource_t*>*/ *resources = NULL;
 451 
 452     if ((attr_id == NULL) && !force) {
 453         find_resource_attr(out, cib, XML_ATTR_ID,
 454                            pe__const_top_resource(rsc, false)->id, NULL,
 455                            NULL, NULL, attr_name, NULL);
 456     }
 457 
 458     if (pcmk__str_eq(attr_set_type, XML_TAG_META_SETS, pcmk__str_casei)) {
 459         resources = find_matching_attr_resources(out, rsc, requested_name,
 460                                                  attr_set, attr_set_type,
 461                                                  attr_id, attr_name, cib,
 462                                                  "delete", force);
 463 
 464     } else if (pcmk__str_eq(attr_set_type, ATTR_SET_ELEMENT, pcmk__str_none)) {
 465         xml_remove_prop(rsc->xml, attr_name);
 466         CRM_ASSERT(cib != NULL);
 467         rc = cib->cmds->replace(cib, XML_CIB_TAG_RESOURCES, rsc->xml,
 468                                 cib_options);
 469         rc = pcmk_legacy2rc(rc);
 470         if (rc == pcmk_rc_ok) {
 471             out->info(out, "Deleted attribute: %s", attr_name);
 472         }
 473         return rc;
 474 
 475     } else {
 476         resources = g_list_append(resources, rsc);
 477     }
 478 
 479     for (GList *iter = resources; iter != NULL; iter = iter->next) {
 480         char *lookup_id = NULL;
 481         xmlNode *xml_obj = NULL;
 482         char *found_attr_id = NULL;
 483         const char *rsc_attr_id = attr_id;
 484 
 485         rsc = (pe_resource_t *) iter->data;
 486 
 487         lookup_id = clone_strip(rsc->id);
 488         rc = find_resource_attr(out, cib, XML_ATTR_ID, lookup_id, attr_set_type,
 489                                 attr_set, attr_id, attr_name, &found_attr_id);
 490         switch (rc) {
 491             case pcmk_rc_ok:
 492                 break;
 493 
 494             case ENXIO:
 495                 free(lookup_id);
 496                 rc = pcmk_rc_ok;
 497                 continue;
 498 
 499             default:
 500                 free(lookup_id);
 501                 g_list_free(resources);
 502                 return rc;
 503         }
 504 
 505         if (rsc_attr_id == NULL) {
 506             rsc_attr_id = found_attr_id;
 507         }
 508 
 509         xml_obj = crm_create_nvpair_xml(NULL, rsc_attr_id, attr_name, NULL);
 510         crm_log_xml_debug(xml_obj, "Delete");
 511 
 512         CRM_ASSERT(cib);
 513         rc = cib->cmds->remove(cib, XML_CIB_TAG_RESOURCES, xml_obj,
 514                                cib_options);
 515         rc = pcmk_legacy2rc(rc);
 516 
 517         if (rc == pcmk_rc_ok) {
 518             out->info(out, "Deleted '%s' option: id=%s%s%s%s%s",
 519                       lookup_id, found_attr_id,
 520                       ((attr_set == NULL)? "" : " set="),
 521                       pcmk__s(attr_set, ""),
 522                       ((attr_name == NULL)? "" : " name="),
 523                       pcmk__s(attr_name, ""));
 524         }
 525 
 526         free(lookup_id);
 527         free_xml(xml_obj);
 528         free(found_attr_id);
 529     }
 530     g_list_free(resources);
 531     return rc;
 532 }
 533 
 534 // \return Standard Pacemaker return code
 535 static int
 536 send_lrm_rsc_op(pcmk_ipc_api_t *controld_api, bool do_fail_resource,
     /*  */
 537                 const char *host_uname, const char *rsc_id, pe_working_set_t *data_set)
 538 {
 539     pcmk__output_t *out = data_set->priv;
 540     const char *router_node = host_uname;
 541     const char *rsc_api_id = NULL;
 542     const char *rsc_long_id = NULL;
 543     const char *rsc_class = NULL;
 544     const char *rsc_provider = NULL;
 545     const char *rsc_type = NULL;
 546     bool cib_only = false;
 547     pe_resource_t *rsc = pe_find_resource(data_set->resources, rsc_id);
 548 
 549     if (rsc == NULL) {
 550         out->err(out, "Resource %s not found", rsc_id);
 551         return ENXIO;
 552 
 553     } else if (rsc->variant != pe_native) {
 554         out->err(out, "We can only process primitive resources, not %s", rsc_id);
 555         return EINVAL;
 556     }
 557 
 558     rsc_class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
 559     rsc_provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER),
 560     rsc_type = crm_element_value(rsc->xml, XML_ATTR_TYPE);
 561     if ((rsc_class == NULL) || (rsc_type == NULL)) {
 562         out->err(out, "Resource %s does not have a class and type", rsc_id);
 563         return EINVAL;
 564     }
 565 
 566     {
 567         pe_node_t *node = pe_find_node(data_set->nodes, host_uname);
 568 
 569         if (node == NULL) {
 570             out->err(out, "Node %s not found", host_uname);
 571             return pcmk_rc_node_unknown;
 572         }
 573 
 574         if (!(node->details->online)) {
 575             if (do_fail_resource) {
 576                 out->err(out, "Node %s is not online", host_uname);
 577                 return ENOTCONN;
 578             } else {
 579                 cib_only = true;
 580             }
 581         }
 582         if (!cib_only && pe__is_guest_or_remote_node(node)) {
 583             node = pe__current_node(node->details->remote_rsc);
 584             if (node == NULL) {
 585                 out->err(out, "No cluster connection to Pacemaker Remote node %s detected",
 586                          host_uname);
 587                 return ENOTCONN;
 588             }
 589             router_node = node->details->uname;
 590         }
 591     }
 592 
 593     if (rsc->clone_name) {
 594         rsc_api_id = rsc->clone_name;
 595         rsc_long_id = rsc->id;
 596     } else {
 597         rsc_api_id = rsc->id;
 598     }
 599     if (do_fail_resource) {
 600         return pcmk_controld_api_fail(controld_api, host_uname, router_node,
 601                                       rsc_api_id, rsc_long_id,
 602                                       rsc_class, rsc_provider, rsc_type);
 603     } else {
 604         return pcmk_controld_api_refresh(controld_api, host_uname, router_node,
 605                                          rsc_api_id, rsc_long_id, rsc_class,
 606                                          rsc_provider, rsc_type, cib_only);
 607     }
 608 }
 609 
 610 /*!
 611  * \internal
 612  * \brief Get resource name as used in failure-related node attributes
 613  *
 614  * \param[in] rsc  Resource to check
 615  *
 616  * \return Newly allocated string containing resource's fail name
 617  * \note The caller is responsible for freeing the result.
 618  */
 619 static inline char *
 620 rsc_fail_name(const pe_resource_t *rsc)
     /*  */
 621 {
 622     const char *name = (rsc->clone_name? rsc->clone_name : rsc->id);
 623 
 624     return pcmk_is_set(rsc->flags, pe_rsc_unique)? strdup(name) : clone_strip(name);
 625 }
 626 
 627 // \return Standard Pacemaker return code
 628 static int
 629 clear_rsc_history(pcmk_ipc_api_t *controld_api, const char *host_uname,
     /*  */
 630                   const char *rsc_id, pe_working_set_t *data_set)
 631 {
 632     int rc = pcmk_rc_ok;
 633 
 634     /* Erase the resource's entire LRM history in the CIB, even if we're only
 635      * clearing a single operation's fail count. If we erased only entries for a
 636      * single operation, we might wind up with a wrong idea of the current
 637      * resource state, and we might not re-probe the resource.
 638      */
 639     rc = send_lrm_rsc_op(controld_api, false, host_uname, rsc_id, data_set);
 640     if (rc != pcmk_rc_ok) {
 641         return rc;
 642     }
 643 
 644     crm_trace("Processing %d mainloop inputs",
 645               pcmk_controld_api_replies_expected(controld_api));
 646     while (g_main_context_iteration(NULL, FALSE)) {
 647         crm_trace("Processed mainloop input, %d still remaining",
 648                   pcmk_controld_api_replies_expected(controld_api));
 649     }
 650     return rc;
 651 }
 652 
 653 // \return Standard Pacemaker return code
 654 static int
 655 clear_rsc_failures(pcmk__output_t *out, pcmk_ipc_api_t *controld_api,
     /*  */
 656                    const char *node_name, const char *rsc_id, const char *operation,
 657                    const char *interval_spec, pe_working_set_t *data_set)
 658 {
 659     int rc = pcmk_rc_ok;
 660     const char *failed_value = NULL;
 661     const char *failed_id = NULL;
 662     const char *interval_ms_s = NULL;
 663     GHashTable *rscs = NULL;
 664     GHashTableIter iter;
 665 
 666     /* Create a hash table to use as a set of resources to clean. This lets us
 667      * clean each resource only once (per node) regardless of how many failed
 668      * operations it has.
 669      */
 670     rscs = pcmk__strkey_table(NULL, NULL);
 671 
 672     // Normalize interval to milliseconds for comparison to history entry
 673     if (operation) {
 674         interval_ms_s = crm_strdup_printf("%u",
 675                                           crm_parse_interval_spec(interval_spec));
 676     }
 677 
 678     for (xmlNode *xml_op = pcmk__xml_first_child(data_set->failed);
 679          xml_op != NULL;
 680          xml_op = pcmk__xml_next(xml_op)) {
 681 
 682         failed_id = crm_element_value(xml_op, XML_LRM_ATTR_RSCID);
 683         if (failed_id == NULL) {
 684             // Malformed history entry, should never happen
 685             continue;
 686         }
 687 
 688         // No resource specified means all resources match
 689         if (rsc_id) {
 690             pe_resource_t *fail_rsc = pe_find_resource_with_flags(data_set->resources,
 691                                                                   failed_id,
 692                                                                   pe_find_renamed|pe_find_anon);
 693 
 694             if (!fail_rsc || !pcmk__str_eq(rsc_id, fail_rsc->id, pcmk__str_casei)) {
 695                 continue;
 696             }
 697         }
 698 
 699         // Host name should always have been provided by this point
 700         failed_value = crm_element_value(xml_op, XML_ATTR_UNAME);
 701         if (!pcmk__str_eq(node_name, failed_value, pcmk__str_casei)) {
 702             continue;
 703         }
 704 
 705         // No operation specified means all operations match
 706         if (operation) {
 707             failed_value = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
 708             if (!pcmk__str_eq(operation, failed_value, pcmk__str_casei)) {
 709                 continue;
 710             }
 711 
 712             // Interval (if operation was specified) defaults to 0 (not all)
 713             failed_value = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL_MS);
 714             if (!pcmk__str_eq(interval_ms_s, failed_value, pcmk__str_casei)) {
 715                 continue;
 716             }
 717         }
 718 
 719         g_hash_table_add(rscs, (gpointer) failed_id);
 720     }
 721 
 722     g_hash_table_iter_init(&iter, rscs);
 723     while (g_hash_table_iter_next(&iter, (gpointer *) &failed_id, NULL)) {
 724         crm_debug("Erasing failures of %s on %s", failed_id, node_name);
 725         rc = clear_rsc_history(controld_api, node_name, failed_id, data_set);
 726         if (rc != pcmk_rc_ok) {
 727             return rc;
 728         }
 729     }
 730     g_hash_table_destroy(rscs);
 731     return rc;
 732 }
 733 
 734 // \return Standard Pacemaker return code
 735 static int
 736 clear_rsc_fail_attrs(const pe_resource_t *rsc, const char *operation,
     /*  */
 737                      const char *interval_spec, const pe_node_t *node)
 738 {
 739     int rc = pcmk_rc_ok;
 740     int attr_options = pcmk__node_attr_none;
 741     char *rsc_name = rsc_fail_name(rsc);
 742 
 743     if (pe__is_guest_or_remote_node(node)) {
 744         attr_options |= pcmk__node_attr_remote;
 745     }
 746 
 747     rc = pcmk__attrd_api_clear_failures(NULL, node->details->uname, rsc_name,
 748                                         operation, interval_spec, NULL,
 749                                         attr_options);
 750     free(rsc_name);
 751     return rc;
 752 }
 753 
 754 // \return Standard Pacemaker return code
 755 int
 756 cli_resource_delete(pcmk_ipc_api_t *controld_api, const char *host_uname,
     /*  */
 757                     const pe_resource_t *rsc, const char *operation,
 758                     const char *interval_spec, bool just_failures,
 759                     pe_working_set_t *data_set, gboolean force)
 760 {
 761     pcmk__output_t *out = data_set->priv;
 762     int rc = pcmk_rc_ok;
 763     pe_node_t *node = NULL;
 764 
 765     if (rsc == NULL) {
 766         return ENXIO;
 767 
 768     } else if (rsc->children) {
 769 
 770         for (const GList *lpc = rsc->children; lpc != NULL; lpc = lpc->next) {
 771             const pe_resource_t *child = (const pe_resource_t *) lpc->data;
 772 
 773             rc = cli_resource_delete(controld_api, host_uname, child, operation,
 774                                      interval_spec, just_failures, data_set, force);
 775             if (rc != pcmk_rc_ok) {
 776                 return rc;
 777             }
 778         }
 779         return pcmk_rc_ok;
 780 
 781     } else if (host_uname == NULL) {
 782         GList *lpc = NULL;
 783         GList *nodes = g_hash_table_get_values(rsc->known_on);
 784 
 785         if(nodes == NULL && force) {
 786             nodes = pcmk__copy_node_list(data_set->nodes, false);
 787 
 788         } else if(nodes == NULL && rsc->exclusive_discover) {
 789             GHashTableIter iter;
 790             pe_node_t *node = NULL;
 791 
 792             g_hash_table_iter_init(&iter, rsc->allowed_nodes);
 793             while (g_hash_table_iter_next(&iter, NULL, (void**)&node)) {
 794                 if(node->weight >= 0) {
 795                     nodes = g_list_prepend(nodes, node);
 796                 }
 797             }
 798 
 799         } else if(nodes == NULL) {
 800             nodes = g_hash_table_get_values(rsc->allowed_nodes);
 801         }
 802 
 803         for (lpc = nodes; lpc != NULL; lpc = lpc->next) {
 804             node = (pe_node_t *) lpc->data;
 805 
 806             if (node->details->online) {
 807                 rc = cli_resource_delete(controld_api, node->details->uname, rsc,
 808                                          operation, interval_spec, just_failures,
 809                                          data_set, force);
 810             }
 811             if (rc != pcmk_rc_ok) {
 812                 g_list_free(nodes);
 813                 return rc;
 814             }
 815         }
 816 
 817         g_list_free(nodes);
 818         return pcmk_rc_ok;
 819     }
 820 
 821     node = pe_find_node(data_set->nodes, host_uname);
 822 
 823     if (node == NULL) {
 824         out->err(out, "Unable to clean up %s because node %s not found",
 825                  rsc->id, host_uname);
 826         return ENODEV;
 827     }
 828 
 829     if (!node->details->rsc_discovery_enabled) {
 830         out->err(out, "Unable to clean up %s because resource discovery disabled on %s",
 831                  rsc->id, host_uname);
 832         return EOPNOTSUPP;
 833     }
 834 
 835     if (controld_api == NULL) {
 836         out->err(out, "Dry run: skipping clean-up of %s on %s due to CIB_file",
 837                  rsc->id, host_uname);
 838         return pcmk_rc_ok;
 839     }
 840 
 841     rc = clear_rsc_fail_attrs(rsc, operation, interval_spec, node);
 842     if (rc != pcmk_rc_ok) {
 843         out->err(out, "Unable to clean up %s failures on %s: %s",
 844                  rsc->id, host_uname, pcmk_rc_str(rc));
 845         return rc;
 846     }
 847 
 848     if (just_failures) {
 849         rc = clear_rsc_failures(out, controld_api, host_uname, rsc->id, operation,
 850                                 interval_spec, data_set);
 851     } else {
 852         rc = clear_rsc_history(controld_api, host_uname, rsc->id, data_set);
 853     }
 854     if (rc != pcmk_rc_ok) {
 855         out->err(out, "Cleaned %s failures on %s, but unable to clean history: %s",
 856                  rsc->id, host_uname, pcmk_strerror(rc));
 857     } else {
 858         out->info(out, "Cleaned up %s on %s", rsc->id, host_uname);
 859     }
 860     return rc;
 861 }
 862 
 863 // \return Standard Pacemaker return code
 864 int
 865 cli_cleanup_all(pcmk_ipc_api_t *controld_api, const char *node_name,
     /*  */
 866                 const char *operation, const char *interval_spec,
 867                 pe_working_set_t *data_set)
 868 {
 869     pcmk__output_t *out = data_set->priv;
 870     int rc = pcmk_rc_ok;
 871     int attr_options = pcmk__node_attr_none;
 872     const char *display_name = node_name? node_name : "all nodes";
 873 
 874     if (controld_api == NULL) {
 875         out->info(out, "Dry run: skipping clean-up of %s due to CIB_file",
 876                   display_name);
 877         return rc;
 878     }
 879 
 880     if (node_name) {
 881         pe_node_t *node = pe_find_node(data_set->nodes, node_name);
 882 
 883         if (node == NULL) {
 884             out->err(out, "Unknown node: %s", node_name);
 885             return ENXIO;
 886         }
 887         if (pe__is_guest_or_remote_node(node)) {
 888             attr_options |= pcmk__node_attr_remote;
 889         }
 890     }
 891 
 892     rc = pcmk__attrd_api_clear_failures(NULL, node_name, NULL, operation,
 893                                         interval_spec, NULL, attr_options);
 894     if (rc != pcmk_rc_ok) {
 895         out->err(out, "Unable to clean up all failures on %s: %s",
 896                  display_name, pcmk_rc_str(rc));
 897         return rc;
 898     }
 899 
 900     if (node_name) {
 901         rc = clear_rsc_failures(out, controld_api, node_name, NULL,
 902                                 operation, interval_spec, data_set);
 903         if (rc != pcmk_rc_ok) {
 904             out->err(out, "Cleaned all resource failures on %s, but unable to clean history: %s",
 905                      node_name, pcmk_strerror(rc));
 906             return rc;
 907         }
 908     } else {
 909         for (GList *iter = data_set->nodes; iter; iter = iter->next) {
 910             pe_node_t *node = (pe_node_t *) iter->data;
 911 
 912             rc = clear_rsc_failures(out, controld_api, node->details->uname, NULL,
 913                                     operation, interval_spec, data_set);
 914             if (rc != pcmk_rc_ok) {
 915                 out->err(out, "Cleaned all resource failures on all nodes, but unable to clean history: %s",
 916                          pcmk_strerror(rc));
 917                 return rc;
 918             }
 919         }
 920     }
 921 
 922     out->info(out, "Cleaned up all resources on %s", display_name);
 923     return rc;
 924 }
 925 
 926 static void
 927 check_role(resource_checks_t *checks)
     /*  */
 928 {
 929     const char *role_s = g_hash_table_lookup(checks->rsc->meta,
 930                                              XML_RSC_ATTR_TARGET_ROLE);
 931 
 932     if (role_s == NULL) {
 933         return;
 934     }
 935     switch (text2role(role_s)) {
 936         case RSC_ROLE_STOPPED:
 937             checks->flags |= rsc_remain_stopped;
 938             break;
 939 
 940         case RSC_ROLE_UNPROMOTED:
 941             if (pcmk_is_set(pe__const_top_resource(checks->rsc, false)->flags,
 942                             pe_rsc_promotable)) {
 943                 checks->flags |= rsc_unpromotable;
 944             }
 945             break;
 946 
 947         default:
 948             break;
 949     }
 950 }
 951 
 952 static void
 953 check_managed(resource_checks_t *checks)
     /*  */
 954 {
 955     const char *managed_s = g_hash_table_lookup(checks->rsc->meta,
 956                                                 XML_RSC_ATTR_MANAGED);
 957 
 958     if ((managed_s != NULL) && !crm_is_true(managed_s)) {
 959         checks->flags |= rsc_unmanaged;
 960     }
 961 }
 962 
 963 static void
 964 check_locked(resource_checks_t *checks)
     /*  */
 965 {
 966     if (checks->rsc->lock_node != NULL) {
 967         checks->flags |= rsc_locked;
 968         checks->lock_node = checks->rsc->lock_node->details->uname;
 969     }
 970 }
 971 
 972 static bool
 973 node_is_unhealthy(pe_node_t *node)
     /*  */
 974 {
 975     switch (pe__health_strategy(node->details->data_set)) {
 976         case pcmk__health_strategy_none:
 977             break;
 978 
 979         case pcmk__health_strategy_no_red:
 980             if (pe__node_health(node) < 0) {
 981                 return true;
 982             }
 983             break;
 984 
 985         case pcmk__health_strategy_only_green:
 986             if (pe__node_health(node) <= 0) {
 987                 return true;
 988             }
 989             break;
 990 
 991         case pcmk__health_strategy_progressive:
 992         case pcmk__health_strategy_custom:
 993             /* @TODO These are finite scores, possibly with rules, and possibly
 994              * combining with other scores, so attributing these as a cause is
 995              * nontrivial.
 996              */
 997             break;
 998     }
 999     return false;
1000 }
1001 
1002 static void
1003 check_node_health(resource_checks_t *checks, pe_node_t *node)
     /*  */
1004 {
1005     if (node == NULL) {
1006         GHashTableIter iter;
1007         bool allowed = false;
1008         bool all_nodes_unhealthy = true;
1009 
1010         g_hash_table_iter_init(&iter, checks->rsc->allowed_nodes);
1011         while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
1012             allowed = true;
1013             if (!node_is_unhealthy(node)) {
1014                 all_nodes_unhealthy = false;
1015                 break;
1016             }
1017         }
1018         if (allowed && all_nodes_unhealthy) {
1019             checks->flags |= rsc_node_health;
1020         }
1021 
1022     } else if (node_is_unhealthy(node)) {
1023         checks->flags |= rsc_node_health;
1024     }
1025 }
1026 
1027 int
1028 cli_resource_check(pcmk__output_t *out, pe_resource_t *rsc, pe_node_t *node)
     /*  */
1029 {
1030     resource_checks_t checks = { .rsc = rsc };
1031 
1032     check_role(&checks);
1033     check_managed(&checks);
1034     check_locked(&checks);
1035     check_node_health(&checks, node);
1036 
1037     return out->message(out, "resource-check-list", &checks);
1038 }
1039 
1040 // \return Standard Pacemaker return code
1041 int
1042 cli_resource_fail(pcmk_ipc_api_t *controld_api, const char *host_uname,
     /*  */
1043                   const char *rsc_id, pe_working_set_t *data_set)
1044 {
1045     crm_notice("Failing %s on %s", rsc_id, host_uname);
1046     return send_lrm_rsc_op(controld_api, true, host_uname, rsc_id, data_set);
1047 }
1048 
1049 static GHashTable *
1050 generate_resource_params(pe_resource_t *rsc, pe_node_t *node,
     /*  */
1051                          pe_working_set_t *data_set)
1052 {
1053     GHashTable *params = NULL;
1054     GHashTable *meta = NULL;
1055     GHashTable *combined = NULL;
1056     GHashTableIter iter;
1057     char *key = NULL;
1058     char *value = NULL;
1059 
1060     combined = pcmk__strkey_table(free, free);
1061 
1062     params = pe_rsc_params(rsc, node, data_set);
1063     if (params != NULL) {
1064         g_hash_table_iter_init(&iter, params);
1065         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
1066             g_hash_table_insert(combined, strdup(key), strdup(value));
1067         }
1068     }
1069 
1070     meta = pcmk__strkey_table(free, free);
1071     get_meta_attributes(meta, rsc, node, data_set);
1072     if (meta != NULL) {
1073         g_hash_table_iter_init(&iter, meta);
1074         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
1075             char *crm_name = crm_meta_name(key);
1076 
1077             g_hash_table_insert(combined, crm_name, strdup(value));
1078         }
1079         g_hash_table_destroy(meta);
1080     }
1081 
1082     return combined;
1083 }
1084 
1085 bool resource_is_running_on(pe_resource_t *rsc, const char *host)
     /*  */
1086 {
1087     bool found = true;
1088     GList *hIter = NULL;
1089     GList *hosts = NULL;
1090 
1091     if (rsc == NULL) {
1092         return false;
1093     }
1094 
1095     rsc->fns->location(rsc, &hosts, TRUE);
1096     for (hIter = hosts; host != NULL && hIter != NULL; hIter = hIter->next) {
1097         pe_node_t *node = (pe_node_t *) hIter->data;
1098 
1099         if (pcmk__strcase_any_of(host, node->details->uname, node->details->id, NULL)) {
1100             crm_trace("Resource %s is running on %s\n", rsc->id, host);
1101             goto done;
1102         }
1103     }
1104 
1105     if (host != NULL) {
1106         crm_trace("Resource %s is not running on: %s\n", rsc->id, host);
1107         found = false;
1108 
1109     } else if(host == NULL && hosts == NULL) {
1110         crm_trace("Resource %s is not running\n", rsc->id);
1111         found = false;
1112     }
1113 
1114   done:
1115     g_list_free(hosts);
1116     return found;
1117 }
1118 
1119 /*!
1120  * \internal
1121  * \brief Create a list of all resources active on host from a given list
1122  *
1123  * \param[in] host      Name of host to check whether resources are active
1124  * \param[in] rsc_list  List of resources to check
1125  *
1126  * \return New list of resources from list that are active on host
1127  */
1128 static GList *
1129 get_active_resources(const char *host, GList *rsc_list)
     /*  */
1130 {
1131     GList *rIter = NULL;
1132     GList *active = NULL;
1133 
1134     for (rIter = rsc_list; rIter != NULL; rIter = rIter->next) {
1135         pe_resource_t *rsc = (pe_resource_t *) rIter->data;
1136 
1137         /* Expand groups to their members, because if we're restarting a member
1138          * other than the first, we can't otherwise tell which resources are
1139          * stopping and starting.
1140          */
1141         if (rsc->variant == pe_group) {
1142             active = g_list_concat(active,
1143                                    get_active_resources(host, rsc->children));
1144         } else if (resource_is_running_on(rsc, host)) {
1145             active = g_list_append(active, strdup(rsc->id));
1146         }
1147     }
1148     return active;
1149 }
1150 
1151 static void dump_list(GList *items, const char *tag) 
     /*  */
1152 {
1153     int lpc = 0;
1154     GList *item = NULL;
1155 
1156     for (item = items; item != NULL; item = item->next) {
1157         crm_trace("%s[%d]: %s", tag, lpc, (char*)item->data);
1158         lpc++;
1159     }
1160 }
1161 
1162 static void display_list(pcmk__output_t *out, GList *items, const char *tag)
     /*  */
1163 {
1164     GList *item = NULL;
1165 
1166     for (item = items; item != NULL; item = item->next) {
1167         out->info(out, "%s%s", tag, (const char *)item->data);
1168     }
1169 }
1170 
1171 /*!
1172  * \internal
1173  * \brief Upgrade XML to latest schema version and use it as working set input
1174  *
1175  * This also updates the working set timestamp to the current time.
1176  *
1177  * \param[in,out] data_set  Working set instance to update
1178  * \param[in,out] xml       XML to use as input
1179  *
1180  * \return Standard Pacemaker return code
1181  * \note On success, caller is responsible for freeing memory allocated for
1182  *       data_set->now.
1183  * \todo This follows the example of other callers of cli_config_update()
1184  *       and returns ENOKEY ("Required key not available") if that fails,
1185  *       but perhaps pcmk_rc_schema_validation would be better in that case.
1186  */
1187 int
1188 update_working_set_xml(pe_working_set_t *data_set, xmlNode **xml)
     /*  */
1189 {
1190     if (cli_config_update(xml, NULL, FALSE) == FALSE) {
1191         return ENOKEY;
1192     }
1193     data_set->input = *xml;
1194     data_set->now = crm_time_new(NULL);
1195     return pcmk_rc_ok;
1196 }
1197 
1198 /*!
1199  * \internal
1200  * \brief Update a working set's XML input based on a CIB query
1201  *
1202  * \param[in] data_set   Data set instance to initialize
1203  * \param[in] cib        Connection to the CIB manager
1204  *
1205  * \return Standard Pacemaker return code
1206  * \note On success, caller is responsible for freeing memory allocated for
1207  *       data_set->input and data_set->now.
1208  */
1209 static int
1210 update_working_set_from_cib(pcmk__output_t *out, pe_working_set_t * data_set,
     /*  */
1211                             cib_t *cib)
1212 {
1213     xmlNode *cib_xml_copy = NULL;
1214     int rc = pcmk_rc_ok;
1215 
1216     rc = cib->cmds->query(cib, NULL, &cib_xml_copy, cib_scope_local | cib_sync_call);
1217     rc = pcmk_legacy2rc(rc);
1218 
1219     if (rc != pcmk_rc_ok) {
1220         out->err(out, "Could not obtain the current CIB: %s (%d)", pcmk_strerror(rc), rc);
1221         return rc;
1222     }
1223     rc = update_working_set_xml(data_set, &cib_xml_copy);
1224     if (rc != pcmk_rc_ok) {
1225         out->err(out, "Could not upgrade the current CIB XML");
1226         free_xml(cib_xml_copy);
1227         return rc;
1228     }
1229 
1230     return rc;
1231 }
1232 
1233 // \return Standard Pacemaker return code
1234 static int
1235 update_dataset(cib_t *cib, pe_working_set_t * data_set, bool simulate)
     /*  */
1236 {
1237     char *pid = NULL;
1238     char *shadow_file = NULL;
1239     cib_t *shadow_cib = NULL;
1240     int rc = pcmk_rc_ok;
1241 
1242     pcmk__output_t *out = data_set->priv;
1243 
1244     pe_reset_working_set(data_set);
1245     pe__set_working_set_flags(data_set, pe_flag_no_counts|pe_flag_no_compat);
1246     rc = update_working_set_from_cib(out, data_set, cib);
1247     if (rc != pcmk_rc_ok) {
1248         return rc;
1249     }
1250 
1251     if(simulate) {
1252         bool prev_quiet = false;
1253 
1254         pid = pcmk__getpid_s();
1255         shadow_cib = cib_shadow_new(pid);
1256         shadow_file = get_shadow_file(pid);
1257 
1258         if (shadow_cib == NULL) {
1259             out->err(out, "Could not create shadow cib: '%s'", pid);
1260             rc = ENXIO;
1261             goto done;
1262         }
1263 
1264         rc = write_xml_file(data_set->input, shadow_file, FALSE);
1265 
1266         if (rc < 0) {
1267             out->err(out, "Could not populate shadow cib: %s (%d)", pcmk_strerror(rc), rc);
1268             goto done;
1269         }
1270 
1271         rc = shadow_cib->cmds->signon(shadow_cib, crm_system_name, cib_command);
1272         rc = pcmk_legacy2rc(rc);
1273 
1274         if (rc != pcmk_rc_ok) {
1275             out->err(out, "Could not connect to shadow cib: %s (%d)", pcmk_strerror(rc), rc);
1276             goto done;
1277         }
1278 
1279         pcmk__schedule_actions(data_set->input,
1280                                pe_flag_no_counts|pe_flag_no_compat, data_set);
1281 
1282         prev_quiet = out->is_quiet(out);
1283         out->quiet = true;
1284         pcmk__simulate_transition(data_set, shadow_cib, NULL);
1285         out->quiet = prev_quiet;
1286 
1287         rc = update_dataset(shadow_cib, data_set, false);
1288 
1289     } else {
1290         cluster_status(data_set);
1291     }
1292 
1293   done:
1294     /* Do not free data_set->input here, we need rsc->xml to be valid later on */
1295     cib_delete(shadow_cib);
1296     free(pid);
1297 
1298     if(shadow_file) {
1299         unlink(shadow_file);
1300         free(shadow_file);
1301     }
1302 
1303     return rc;
1304 }
1305 
1306 static int
1307 max_delay_for_resource(pe_working_set_t * data_set, pe_resource_t *rsc) 
     /*  */
1308 {
1309     int delay = 0;
1310     int max_delay = 0;
1311 
1312     if(rsc && rsc->children) {
1313         GList *iter = NULL;
1314 
1315         for(iter = rsc->children; iter; iter = iter->next) {
1316             pe_resource_t *child = (pe_resource_t *)iter->data;
1317 
1318             delay = max_delay_for_resource(data_set, child);
1319             if(delay > max_delay) {
1320                 double seconds = delay / 1000.0;
1321                 crm_trace("Calculated new delay of %.1fs due to %s", seconds, child->id);
1322                 max_delay = delay;
1323             }
1324         }
1325 
1326     } else if(rsc) {
1327         char *key = crm_strdup_printf("%s_%s_0", rsc->id, RSC_STOP);
1328         pe_action_t *stop = custom_action(rsc, key, RSC_STOP, NULL, TRUE, FALSE, data_set);
1329         const char *value = g_hash_table_lookup(stop->meta, XML_ATTR_TIMEOUT);
1330         long long result_ll;
1331 
1332         if ((pcmk__scan_ll(value, &result_ll, -1LL) == pcmk_rc_ok)
1333             && (result_ll >= 0) && (result_ll <= INT_MAX)) {
1334             max_delay = (int) result_ll;
1335         } else {
1336             max_delay = -1;
1337         }
1338         pe_free_action(stop);
1339     }
1340 
1341     return max_delay;
1342 }
1343 
1344 static int
1345 max_delay_in(pe_working_set_t * data_set, GList *resources) 
     /*  */
1346 {
1347     int max_delay = 0;
1348     GList *item = NULL;
1349 
1350     for (item = resources; item != NULL; item = item->next) {
1351         int delay = 0;
1352         pe_resource_t *rsc = pe_find_resource(data_set->resources, (const char *)item->data);
1353 
1354         delay = max_delay_for_resource(data_set, rsc);
1355 
1356         if(delay > max_delay) {
1357             double seconds = delay / 1000.0;
1358             crm_trace("Calculated new delay of %.1fs due to %s", seconds, rsc->id);
1359             max_delay = delay;
1360         }
1361     }
1362 
1363     return 5 + (max_delay / 1000);
1364 }
1365 
1366 #define waiting_for_starts(d, r, h) ((d != NULL) || \
1367                                     (!resource_is_running_on((r), (h))))
1368 
1369 /*!
1370  * \internal
1371  * \brief Restart a resource (on a particular host if requested).
1372  *
1373  * \param[in,out] out                 Output object
1374  * \param[in,out] rsc                 The resource to restart
1375  * \param[in]     node                Node to restart resource on (NULL for all)
1376  * \param[in]     move_lifetime       If not NULL, how long constraint should
1377  *                                    remain in effect (as ISO 8601 string)
1378  * \param[in]     timeout_ms          Consider failed if actions do not complete
1379  *                                    in this time (specified in milliseconds,
1380  *                                    but a two-second granularity is actually
1381  *                                    used; if 0, it will be calculated based on
1382  *                                    the resource timeout)
1383  * \param[in,out] cib                 Connection to the CIB manager
1384  * \param[in]     cib_options         Group of enum cib_call_options flags to
1385  *                                    use with CIB calls
1386  * \param[in]     promoted_role_only  If true, limit to promoted instances
1387  * \param[in]     force               If true, apply only to requested instance
1388  *                                    if part of a collective resource
1389  *
1390  * \return Standard Pacemaker return code (exits on certain failures)
1391  */
1392 int
1393 cli_resource_restart(pcmk__output_t *out, pe_resource_t *rsc,
     /*  */
1394                      const pe_node_t *node, const char *move_lifetime,
1395                      int timeout_ms, cib_t *cib, int cib_options,
1396                      gboolean promoted_role_only, gboolean force)
1397 {
1398     int rc = pcmk_rc_ok;
1399     int lpc = 0;
1400     int before = 0;
1401     int step_timeout_s = 0;
1402     int sleep_interval = 2;
1403     int timeout = timeout_ms / 1000;
1404 
1405     bool stop_via_ban = false;
1406     char *rsc_id = NULL;
1407     char *lookup_id = NULL;
1408     char *orig_target_role = NULL;
1409 
1410     GList *list_delta = NULL;
1411     GList *target_active = NULL;
1412     GList *current_active = NULL;
1413     GList *restart_target_active = NULL;
1414 
1415     pe_working_set_t *data_set = NULL;
1416     pe_resource_t *parent = uber_parent(rsc);
1417 
1418     bool running = false;
1419     const char *id = rsc->clone_name ? rsc->clone_name : rsc->id;
1420     const char *host = node ? node->details->uname : NULL;
1421 
1422     /* If the implicit resource or primitive resource of a bundle is given, operate on the
1423      * bundle itself instead.
1424      */
1425     if (pe_rsc_is_bundled(rsc)) {
1426         rsc = parent->parent;
1427     }
1428 
1429     running = resource_is_running_on(rsc, host);
1430 
1431     if (pe_rsc_is_clone(parent) && !running) {
1432         if (pe_rsc_is_unique_clone(parent)) {
1433             lookup_id = strdup(rsc->id);
1434         } else {
1435             lookup_id = clone_strip(rsc->id);
1436         }
1437 
1438         rsc = parent->fns->find_rsc(parent, lookup_id, node, pe_find_any|pe_find_current);
1439         free(lookup_id);
1440         running = resource_is_running_on(rsc, host);
1441     }
1442 
1443     if (!running) {
1444         if (host) {
1445             out->err(out, "%s is not running on %s and so cannot be restarted", id, host);
1446         } else {
1447             out->err(out, "%s is not running anywhere and so cannot be restarted", id);
1448         }
1449         return ENXIO;
1450     }
1451 
1452     rsc_id = strdup(rsc->id);
1453 
1454     if (pe_rsc_is_unique_clone(parent)) {
1455         lookup_id = strdup(rsc->id);
1456     } else {
1457         lookup_id = clone_strip(rsc->id);
1458     }
1459 
1460     if (host) {
1461         if (pe_rsc_is_clone(rsc) || pe_bundle_replicas(rsc)) {
1462             stop_via_ban = true;
1463         } else if (pe_rsc_is_clone(parent)) {
1464             stop_via_ban = true;
1465             free(lookup_id);
1466             lookup_id = strdup(parent->id);
1467         }
1468     }
1469 
1470     /*
1471       grab full cib
1472       determine originally active resources
1473       disable or ban
1474       poll cib and watch for affected resources to get stopped
1475       without --timeout, calculate the stop timeout for each step and wait for that
1476       if we hit --timeout or the service timeout, re-enable or un-ban, report failure and indicate which resources we couldn't take down
1477       if everything stopped, re-enable or un-ban
1478       poll cib and watch for affected resources to get started
1479       without --timeout, calculate the start timeout for each step and wait for that
1480       if we hit --timeout or the service timeout, report (different) failure and indicate which resources we couldn't bring back up
1481       report success
1482 
1483       Optimizations:
1484       - use constraints to determine ordered list of affected resources
1485       - Allow a --no-deps option (aka. --force-restart)
1486     */
1487 
1488     data_set = pe_new_working_set();
1489     if (data_set == NULL) {
1490         crm_perror(LOG_ERR, "Could not allocate working set");
1491         rc = ENOMEM;
1492         goto done;
1493     }
1494 
1495     data_set->priv = out;
1496     rc = update_dataset(cib, data_set, false);
1497 
1498     if(rc != pcmk_rc_ok) {
1499         out->err(out, "Could not get new resource list: %s (%d)", pcmk_strerror(rc), rc);
1500         goto done;
1501     }
1502 
1503     restart_target_active = get_active_resources(host, data_set->resources);
1504     current_active = get_active_resources(host, data_set->resources);
1505 
1506     dump_list(current_active, "Origin");
1507 
1508     if (stop_via_ban) {
1509         /* Stop the clone or bundle instance by banning it from the host */
1510         out->quiet = true;
1511         rc = cli_resource_ban(out, lookup_id, host, move_lifetime, NULL, cib,
1512                               cib_options, promoted_role_only);
1513 
1514     } else {
1515         /* Stop the resource by setting target-role to Stopped.
1516          * Remember any existing target-role so we can restore it later
1517          * (though it only makes any difference if it's Unpromoted).
1518          */
1519 
1520         find_resource_attr(out, cib, XML_NVPAIR_ATTR_VALUE, lookup_id, NULL, NULL,
1521                            NULL, XML_RSC_ATTR_TARGET_ROLE, &orig_target_role);
1522         rc = cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
1523                                            NULL, XML_RSC_ATTR_TARGET_ROLE,
1524                                            RSC_STOPPED, FALSE, cib, cib_options,
1525                                            force);
1526     }
1527     if(rc != pcmk_rc_ok) {
1528         out->err(out, "Could not set target-role for %s: %s (%d)", rsc_id, pcmk_strerror(rc), rc);
1529         if (current_active != NULL) {
1530             g_list_free_full(current_active, free);
1531             current_active = NULL;
1532         }
1533         if (restart_target_active != NULL) {
1534             g_list_free_full(restart_target_active, free);
1535             restart_target_active = NULL;
1536         }
1537         goto done;
1538     }
1539 
1540     rc = update_dataset(cib, data_set, true);
1541     if(rc != pcmk_rc_ok) {
1542         out->err(out, "Could not determine which resources would be stopped");
1543         goto failure;
1544     }
1545 
1546     target_active = get_active_resources(host, data_set->resources);
1547     dump_list(target_active, "Target");
1548 
1549     list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
1550     out->info(out, "Waiting for %d resources to stop:", g_list_length(list_delta));
1551     display_list(out, list_delta, " * ");
1552 
1553     step_timeout_s = timeout / sleep_interval;
1554     while (list_delta != NULL) {
1555         before = g_list_length(list_delta);
1556         if(timeout_ms == 0) {
1557             step_timeout_s = max_delay_in(data_set, list_delta) / sleep_interval;
1558         }
1559 
1560         /* We probably don't need the entire step timeout */
1561         for(lpc = 0; (lpc < step_timeout_s) && (list_delta != NULL); lpc++) {
1562             sleep(sleep_interval);
1563             if(timeout) {
1564                 timeout -= sleep_interval;
1565                 crm_trace("%ds remaining", timeout);
1566             }
1567             rc = update_dataset(cib, data_set, FALSE);
1568             if(rc != pcmk_rc_ok) {
1569                 out->err(out, "Could not determine which resources were stopped");
1570                 goto failure;
1571             }
1572 
1573             if (current_active != NULL) {
1574                 g_list_free_full(current_active, free);
1575                 current_active = NULL;
1576             }
1577             current_active = get_active_resources(host, data_set->resources);
1578             g_list_free(list_delta);
1579             list_delta = NULL;
1580             list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
1581             dump_list(current_active, "Current");
1582             dump_list(list_delta, "Delta");
1583         }
1584 
1585         crm_trace("%d (was %d) resources remaining", g_list_length(list_delta), before);
1586         if(before == g_list_length(list_delta)) {
1587             /* aborted during stop phase, print the contents of list_delta */
1588             out->err(out, "Could not complete shutdown of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
1589             display_list(out, list_delta, " * ");
1590             rc = ETIME;
1591             goto failure;
1592         }
1593 
1594     }
1595 
1596     if (stop_via_ban) {
1597         rc = cli_resource_clear(lookup_id, host, NULL, cib, cib_options, true, force);
1598 
1599     } else if (orig_target_role) {
1600         rc = cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
1601                                            NULL, XML_RSC_ATTR_TARGET_ROLE,
1602                                            orig_target_role, FALSE, cib,
1603                                            cib_options, force);
1604         free(orig_target_role);
1605         orig_target_role = NULL;
1606     } else {
1607         rc = cli_resource_delete_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
1608                                            NULL, XML_RSC_ATTR_TARGET_ROLE, cib,
1609                                            cib_options, force);
1610     }
1611 
1612     if(rc != pcmk_rc_ok) {
1613         out->err(out, "Could not unset target-role for %s: %s (%d)", rsc_id, pcmk_strerror(rc), rc);
1614         goto done;
1615     }
1616 
1617     if (target_active != NULL) {
1618         g_list_free_full(target_active, free);
1619         target_active = NULL;
1620     }
1621     target_active = restart_target_active;
1622     list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
1623     out->info(out, "Waiting for %d resources to start again:", g_list_length(list_delta));
1624     display_list(out, list_delta, " * ");
1625 
1626     step_timeout_s = timeout / sleep_interval;
1627     while (waiting_for_starts(list_delta, rsc, host)) {
1628         before = g_list_length(list_delta);
1629         if(timeout_ms == 0) {
1630             step_timeout_s = max_delay_in(data_set, list_delta) / sleep_interval;
1631         }
1632 
1633         /* We probably don't need the entire step timeout */
1634         for (lpc = 0; (lpc < step_timeout_s) && waiting_for_starts(list_delta, rsc, host); lpc++) {
1635 
1636             sleep(sleep_interval);
1637             if(timeout) {
1638                 timeout -= sleep_interval;
1639                 crm_trace("%ds remaining", timeout);
1640             }
1641 
1642             rc = update_dataset(cib, data_set, false);
1643             if(rc != pcmk_rc_ok) {
1644                 out->err(out, "Could not determine which resources were started");
1645                 goto failure;
1646             }
1647 
1648             if (current_active != NULL) {
1649                 g_list_free_full(current_active, free);
1650                 current_active = NULL;
1651             }
1652 
1653             /* It's OK if dependent resources moved to a different node,
1654              * so we check active resources on all nodes.
1655              */
1656             current_active = get_active_resources(NULL, data_set->resources);
1657             g_list_free(list_delta);
1658             list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
1659             dump_list(current_active, "Current");
1660             dump_list(list_delta, "Delta");
1661         }
1662 
1663         if(before == g_list_length(list_delta)) {
1664             /* aborted during start phase, print the contents of list_delta */
1665             out->err(out, "Could not complete restart of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
1666             display_list(out, list_delta, " * ");
1667             rc = ETIME;
1668             goto failure;
1669         }
1670 
1671     }
1672 
1673     rc = pcmk_rc_ok;
1674     goto done;
1675 
1676   failure:
1677     if (stop_via_ban) {
1678         cli_resource_clear(lookup_id, host, NULL, cib, cib_options, true, force);
1679     } else if (orig_target_role) {
1680         cli_resource_update_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS, NULL,
1681                                       XML_RSC_ATTR_TARGET_ROLE, orig_target_role,
1682                                       FALSE, cib, cib_options, force);
1683         free(orig_target_role);
1684     } else {
1685         cli_resource_delete_attribute(rsc, rsc_id, NULL, XML_TAG_META_SETS,
1686                                       NULL, XML_RSC_ATTR_TARGET_ROLE, cib,
1687                                       cib_options, force);
1688     }
1689 
1690 done:
1691     if (list_delta != NULL) {
1692         g_list_free(list_delta);
1693     }
1694     if (current_active != NULL) {
1695         g_list_free_full(current_active, free);
1696     }
1697     if (target_active != NULL && (target_active != restart_target_active)) {
1698         g_list_free_full(target_active, free);
1699     }
1700     if (restart_target_active != NULL) {
1701         g_list_free_full(restart_target_active, free);
1702     }
1703     free(rsc_id);
1704     free(lookup_id);
1705     pe_free_working_set(data_set);
1706     return rc;
1707 }
1708 
1709 static inline bool
1710 action_is_pending(const pe_action_t *action)
     /*  */
1711 {
1712     if (pcmk_any_flags_set(action->flags, pe_action_optional|pe_action_pseudo)
1713         || !pcmk_is_set(action->flags, pe_action_runnable)
1714         || pcmk__str_eq("notify", action->task, pcmk__str_casei)) {
1715         return false;
1716     }
1717     return true;
1718 }
1719 
1720 /*!
1721  * \internal
1722  * \brief Check whether any actions in a list are pending
1723  *
1724  * \param[in] actions   List of actions to check
1725  *
1726  * \return true if any actions in the list are pending, otherwise false
1727  */
1728 static bool
1729 actions_are_pending(const GList *actions)
     /*  */
1730 {
1731     for (const GList *action = actions; action != NULL; action = action->next) {
1732         const pe_action_t *a = (const pe_action_t *) action->data;
1733 
1734         if (action_is_pending(a)) {
1735             crm_notice("Waiting for %s (flags=%#.8x)", a->uuid, a->flags);
1736             return true;
1737         }
1738     }
1739     return false;
1740 }
1741 
1742 static void
1743 print_pending_actions(pcmk__output_t *out, GList *actions)
     /*  */
1744 {
1745     GList *action;
1746 
1747     out->info(out, "Pending actions:");
1748     for (action = actions; action != NULL; action = action->next) {
1749         pe_action_t *a = (pe_action_t *) action->data;
1750 
1751         if (!action_is_pending(a)) {
1752             continue;
1753         }
1754 
1755         if (a->node) {
1756             out->info(out, "\tAction %d: %s\ton %s",
1757                       a->id, a->uuid, pe__node_name(a->node));
1758         } else {
1759             out->info(out, "\tAction %d: %s", a->id, a->uuid);
1760         }
1761     }
1762 }
1763 
1764 /* For --wait, timeout (in seconds) to use if caller doesn't specify one */
1765 #define WAIT_DEFAULT_TIMEOUT_S (60 * 60)
1766 
1767 /* For --wait, how long to sleep between cluster state checks */
1768 #define WAIT_SLEEP_S (2)
1769 
1770 /*!
1771  * \internal
1772  * \brief Wait until all pending cluster actions are complete
1773  *
1774  * This waits until either the CIB's transition graph is idle or a timeout is
1775  * reached.
1776  *
1777  * \param[in,out] out          Output object
1778  * \param[in]     timeout_ms   Consider failed if actions do not complete in
1779  *                             this time (specified in milliseconds, but
1780  *                             one-second granularity is actually used; if 0, a
1781  *                             default will be used)
1782  * \param[in,out] cib          Connection to the CIB manager
1783  *
1784  * \return Standard Pacemaker return code
1785  */
1786 int
1787 wait_till_stable(pcmk__output_t *out, int timeout_ms, cib_t * cib)
     /*  */
1788 {
1789     pe_working_set_t *data_set = NULL;
1790     int rc = pcmk_rc_ok;
1791     int timeout_s = timeout_ms? ((timeout_ms + 999) / 1000) : WAIT_DEFAULT_TIMEOUT_S;
1792     time_t expire_time = time(NULL) + timeout_s;
1793     time_t time_diff;
1794     bool printed_version_warning = out->is_quiet(out); // i.e. don't print if quiet
1795 
1796     data_set = pe_new_working_set();
1797     if (data_set == NULL) {
1798         return ENOMEM;
1799     }
1800 
1801     do {
1802 
1803         /* Abort if timeout is reached */
1804         time_diff = expire_time - time(NULL);
1805         if (time_diff > 0) {
1806             crm_info("Waiting up to %lld seconds for cluster actions to complete", (long long) time_diff);
1807         } else {
1808             print_pending_actions(out, data_set->actions);
1809             pe_free_working_set(data_set);
1810             return ETIME;
1811         }
1812         if (rc == pcmk_rc_ok) { /* this avoids sleep on first loop iteration */
1813             sleep(WAIT_SLEEP_S);
1814         }
1815 
1816         /* Get latest transition graph */
1817         pe_reset_working_set(data_set);
1818         rc = update_working_set_from_cib(out, data_set, cib);
1819         if (rc != pcmk_rc_ok) {
1820             pe_free_working_set(data_set);
1821             return rc;
1822         }
1823         pcmk__schedule_actions(data_set->input,
1824                                pe_flag_no_counts|pe_flag_no_compat, data_set);
1825 
1826         if (!printed_version_warning) {
1827             /* If the DC has a different version than the local node, the two
1828              * could come to different conclusions about what actions need to be
1829              * done. Warn the user in this case.
1830              *
1831              * @TODO A possible long-term solution would be to reimplement the
1832              * wait as a new controller operation that would be forwarded to the
1833              * DC. However, that would have potential problems of its own.
1834              */
1835             const char *dc_version = g_hash_table_lookup(data_set->config_hash,
1836                                                          "dc-version");
1837 
1838             if (!pcmk__str_eq(dc_version, PACEMAKER_VERSION "-" BUILD_VERSION, pcmk__str_casei)) {
1839                 out->info(out, "warning: wait option may not work properly in "
1840                           "mixed-version cluster");
1841                 printed_version_warning = true;
1842             }
1843         }
1844 
1845     } while (actions_are_pending(data_set->actions));
1846 
1847     pe_free_working_set(data_set);
1848     return rc;
1849 }
1850 
1851 static const char *
1852 get_action(const char *rsc_action) {
     /*  */
1853     const char *action = NULL;
1854 
1855     if (pcmk__str_eq(rsc_action, "validate", pcmk__str_casei)) {
1856         action = "validate-all";
1857 
1858     } else if (pcmk__str_eq(rsc_action, "force-check", pcmk__str_casei)) {
1859         action = "monitor";
1860 
1861     } else if (pcmk__strcase_any_of(rsc_action, "force-start", "force-stop",
1862                                     "force-demote", "force-promote", NULL)) {
1863         action = rsc_action+6;
1864     } else {
1865         action = rsc_action;
1866     }
1867 
1868     return action;
1869 }
1870 
1871 /*!
1872  * \brief Set up environment variables as expected by resource agents
1873  *
1874  * When the cluster executes resource agents, it adds certain environment
1875  * variables (directly or via resource meta-attributes) expected by some
1876  * resource agents. Add the essential ones that many resource agents expect, so
1877  * the behavior is the same for command-line execution.
1878  *
1879  * \param[in,out] params       Resource parameters that will be passed to agent
1880  * \param[in]     timeout_ms   Action timeout (in milliseconds)
1881  * \param[in]     check_level  OCF check level
1882  * \param[in]     verbosity    Verbosity level
1883  */
1884 static void
1885 set_agent_environment(GHashTable *params, int timeout_ms, int check_level,
     /*  */
1886                       int verbosity)
1887 {
1888     g_hash_table_insert(params, strdup("CRM_meta_timeout"),
1889                         crm_strdup_printf("%d", timeout_ms));
1890 
1891     g_hash_table_insert(params, strdup(XML_ATTR_CRM_VERSION),
1892                         strdup(CRM_FEATURE_SET));
1893 
1894     if (check_level >= 0) {
1895         char *level = crm_strdup_printf("%d", check_level);
1896 
1897         setenv("OCF_CHECK_LEVEL", level, 1);
1898         free(level);
1899     }
1900 
1901     setenv("HA_debug", (verbosity > 0)? "1" : "0", 1);
1902     if (verbosity > 1) {
1903         setenv("OCF_TRACE_RA", "1", 1);
1904     }
1905 
1906     /* A resource agent using the standard ocf-shellfuncs library will not print
1907      * messages to stderr if it doesn't have a controlling terminal (e.g. if
1908      * crm_resource is called via script or ssh). This forces it to do so.
1909      */
1910     setenv("OCF_TRACE_FILE", "/dev/stderr", 0);
1911 }
1912 
1913 /*!
1914  * \internal
1915  * \brief Apply command-line overrides to resource parameters
1916  *
1917  * \param[in,out] params     Parameters to be passed to agent
1918  * \param[in]     overrides  Parameters to override (or NULL if none)
1919  */
1920 static void
1921 apply_overrides(GHashTable *params, GHashTable *overrides)
     /*  */
1922 {
1923     if (overrides != NULL) {
1924         GHashTableIter iter;
1925         char *name = NULL;
1926         char *value = NULL;
1927 
1928         g_hash_table_iter_init(&iter, overrides);
1929         while (g_hash_table_iter_next(&iter, (gpointer *) &name,
1930                                       (gpointer *) &value)) {
1931             g_hash_table_replace(params, strdup(name), strdup(value));
1932         }
1933     }
1934 }
1935 
1936 crm_exit_t
1937 cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc_name,
     /*  */
1938                                  const char *rsc_class, const char *rsc_prov,
1939                                  const char *rsc_type, const char *rsc_action,
1940                                  GHashTable *params, GHashTable *override_hash,
1941                                  int timeout_ms, int resource_verbose, gboolean force,
1942                                  int check_level)
1943 {
1944     const char *class = rsc_class;
1945     const char *action = get_action(rsc_action);
1946     crm_exit_t exit_code = CRM_EX_OK;
1947     svc_action_t *op = NULL;
1948 
1949     // If no timeout was provided, use the same default as the cluster
1950     if (timeout_ms == 0) {
1951         timeout_ms = crm_get_msec(CRM_DEFAULT_OP_TIMEOUT_S);
1952     }
1953 
1954     set_agent_environment(params, timeout_ms, check_level, resource_verbose);
1955     apply_overrides(params, override_hash);
1956 
1957     op = services__create_resource_action(rsc_name? rsc_name : "test",
1958                                           rsc_class, rsc_prov, rsc_type, action,
1959                                           0, timeout_ms, params, 0);
1960     if (op == NULL) {
1961         out->err(out, "Could not execute %s using %s%s%s:%s: %s",
1962                  action, rsc_class, (rsc_prov? ":" : ""),
1963                  (rsc_prov? rsc_prov : ""), rsc_type, strerror(ENOMEM));
1964         g_hash_table_destroy(params);
1965         return CRM_EX_OSERR;
1966     }
1967 
1968     if (pcmk__str_eq(rsc_class, PCMK_RESOURCE_CLASS_SERVICE, pcmk__str_casei)) {
1969         class = resources_find_service_class(rsc_type);
1970     }
1971     if (!pcmk__strcase_any_of(class, PCMK_RESOURCE_CLASS_OCF,
1972                               PCMK_RESOURCE_CLASS_LSB, NULL)) {
1973         services__format_result(op, CRM_EX_UNIMPLEMENT_FEATURE, PCMK_EXEC_ERROR,
1974                                 "Manual execution of the %s standard is "
1975                                 "unsupported", pcmk__s(class, "unspecified"));
1976     }
1977 
1978     if (op->rc != PCMK_OCF_UNKNOWN) {
1979         exit_code = op->rc;
1980         goto done;
1981     }
1982 
1983     services_action_sync(op);
1984 
1985     // Map results to OCF codes for consistent reporting to user
1986     {
1987         enum ocf_exitcode ocf_code = services_result2ocf(class, action, op->rc);
1988 
1989         // Cast variable instead of function return to keep compilers happy
1990         exit_code = (crm_exit_t) ocf_code;
1991     }
1992 
1993 done:
1994     out->message(out, "resource-agent-action", resource_verbose, rsc_class,
1995                  rsc_prov, rsc_type, rsc_name, rsc_action, override_hash,
1996                  exit_code, op->status, services__exit_reason(op),
1997                  op->stdout_data, op->stderr_data);
1998     services_action_free(op);
1999     return exit_code;
2000 }
2001 
2002 crm_exit_t
2003 cli_resource_execute(pe_resource_t *rsc, const char *requested_name,
     /*  */
2004                      const char *rsc_action, GHashTable *override_hash,
2005                      int timeout_ms, cib_t * cib, pe_working_set_t *data_set,
2006                      int resource_verbose, gboolean force, int check_level)
2007 {
2008     pcmk__output_t *out = data_set->priv;
2009     crm_exit_t exit_code = CRM_EX_OK;
2010     const char *rid = NULL;
2011     const char *rtype = NULL;
2012     const char *rprov = NULL;
2013     const char *rclass = NULL;
2014     GHashTable *params = NULL;
2015 
2016     if (pcmk__strcase_any_of(rsc_action, "force-start", "force-demote",
2017                                     "force-promote", NULL)) {
2018         if(pe_rsc_is_clone(rsc)) {
2019             GList *nodes = cli_resource_search(rsc, requested_name, data_set);
2020             if(nodes != NULL && force == FALSE) {
2021                 out->err(out, "It is not safe to %s %s here: the cluster claims it is already active",
2022                          rsc_action, rsc->id);
2023                 out->err(out, "Try setting target-role=Stopped first or specifying "
2024                          "the force option");
2025                 return CRM_EX_UNSAFE;
2026             }
2027 
2028             g_list_free_full(nodes, free);
2029         }
2030     }
2031 
2032     if(pe_rsc_is_clone(rsc)) {
2033         /* Grab the first child resource in the hope it's not a group */
2034         rsc = rsc->children->data;
2035     }
2036 
2037     if(rsc->variant == pe_group) {
2038         out->err(out, "Sorry, the %s option doesn't support group resources", rsc_action);
2039         return CRM_EX_UNIMPLEMENT_FEATURE;
2040     } else if (rsc->variant == pe_container || pe_rsc_is_bundled(rsc)) {
2041         out->err(out, "Sorry, the %s option doesn't support bundled resources", rsc_action);
2042         return CRM_EX_UNIMPLEMENT_FEATURE;
2043     }
2044 
2045     rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
2046     rprov = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
2047     rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);
2048 
2049     params = generate_resource_params(rsc, NULL /* @TODO use local node */,
2050                                       data_set);
2051 
2052     if (timeout_ms == 0) {
2053         timeout_ms = pe_get_configured_timeout(rsc, get_action(rsc_action), data_set);
2054     }
2055 
2056     rid = pe_rsc_is_anon_clone(rsc->parent)? requested_name : rsc->id;
2057 
2058     exit_code = cli_resource_execute_from_params(out, rid, rclass, rprov, rtype, rsc_action,
2059                                                  params, override_hash, timeout_ms,
2060                                                  resource_verbose, force, check_level);
2061     return exit_code;
2062 }
2063 
2064 // \return Standard Pacemaker return code
2065 int
2066 cli_resource_move(const pe_resource_t *rsc, const char *rsc_id,
     /*  */
2067                   const char *host_name, const char *move_lifetime, cib_t *cib,
2068                   int cib_options, pe_working_set_t *data_set,
2069                   gboolean promoted_role_only, gboolean force)
2070 {
2071     pcmk__output_t *out = data_set->priv;
2072     int rc = pcmk_rc_ok;
2073     unsigned int count = 0;
2074     pe_node_t *current = NULL;
2075     pe_node_t *dest = pe_find_node(data_set->nodes, host_name);
2076     bool cur_is_dest = false;
2077 
2078     if (dest == NULL) {
2079         return pcmk_rc_node_unknown;
2080     }
2081 
2082     if (promoted_role_only && !pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
2083         const pe_resource_t *p = pe__const_top_resource(rsc, false);
2084 
2085         if (pcmk_is_set(p->flags, pe_rsc_promotable)) {
2086             out->info(out, "Using parent '%s' for move instead of '%s'.", rsc->id, rsc_id);
2087             rsc_id = p->id;
2088             rsc = p;
2089 
2090         } else {
2091             out->info(out, "Ignoring --promoted option: %s is not promotable",
2092                       rsc_id);
2093             promoted_role_only = FALSE;
2094         }
2095     }
2096 
2097     current = pe__find_active_requires(rsc, &count);
2098 
2099     if (pcmk_is_set(rsc->flags, pe_rsc_promotable)) {
2100         unsigned int promoted_count = 0;
2101         pe_node_t *promoted_node = NULL;
2102 
2103         for (const GList *iter = rsc->children; iter; iter = iter->next) {
2104             const pe_resource_t *child = (const pe_resource_t *) iter->data;
2105             enum rsc_role_e child_role = child->fns->state(child, TRUE);
2106 
2107             if (child_role == RSC_ROLE_PROMOTED) {
2108                 rsc = child;
2109                 promoted_node = pe__current_node(child);
2110                 promoted_count++;
2111             }
2112         }
2113         if (promoted_role_only || (promoted_count != 0)) {
2114             count = promoted_count;
2115             current = promoted_node;
2116         }
2117 
2118     }
2119 
2120     if (count > 1) {
2121         if (pe_rsc_is_clone(rsc)) {
2122             current = NULL;
2123         } else {
2124             return pcmk_rc_multiple;
2125         }
2126     }
2127 
2128     if (current && (current->details == dest->details)) {
2129         cur_is_dest = true;
2130         if (force) {
2131             crm_info("%s is already %s on %s, reinforcing placement with location constraint.",
2132                      rsc_id, promoted_role_only?"promoted":"active",
2133                      pe__node_name(dest));
2134         } else {
2135             return pcmk_rc_already;
2136         }
2137     }
2138 
2139     /* Clear any previous prefer constraints across all nodes. */
2140     cli_resource_clear(rsc_id, NULL, data_set->nodes, cib, cib_options, false, force);
2141 
2142     /* Clear any previous ban constraints on 'dest'. */
2143     cli_resource_clear(rsc_id, dest->details->uname, data_set->nodes, cib,
2144                        cib_options, TRUE, force);
2145 
2146     /* Record an explicit preference for 'dest' */
2147     rc = cli_resource_prefer(out, rsc_id, dest->details->uname, move_lifetime,
2148                              cib, cib_options, promoted_role_only);
2149 
2150     crm_trace("%s%s now prefers %s%s",
2151               rsc->id, (promoted_role_only? " (promoted)" : ""),
2152               pe__node_name(dest), force?"(forced)":"");
2153 
2154     /* only ban the previous location if current location != destination location.
2155      * it is possible to use -M to enforce a location without regard of where the
2156      * resource is currently located */
2157     if (force && !cur_is_dest) {
2158         /* Ban the original location if possible */
2159         if(current) {
2160             (void)cli_resource_ban(out, rsc_id, current->details->uname, move_lifetime,
2161                                    NULL, cib, cib_options, promoted_role_only);
2162 
2163         } else if(count > 1) {
2164             out->info(out, "Resource '%s' is currently %s in %d locations. "
2165                       "One may now move to %s",
2166                       rsc_id, (promoted_role_only? "promoted" : "active"),
2167                       count, pe__node_name(dest));
2168             out->info(out, "To prevent '%s' from being %s at a specific location, "
2169                       "specify a node.",
2170                       rsc_id, (promoted_role_only? "promoted" : "active"));
2171 
2172         } else {
2173             crm_trace("Not banning %s from its current location: not active", rsc_id);
2174         }
2175     }
2176 
2177     return rc;
2178 }